diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f4c0c71..29fb8cfa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,11 +29,17 @@ * `bedtools`: - `bedtools/bedtools_intersect`: Allows one to screen for overlaps between two sets of genomic features (PR #94). - `bedtools/bedtools_sort`: Sorts a feature file (bed/gff/vcf) by chromosome and other criteria (PR #98). + - `bedtools/bedtools_groupby`: Summarizes a dataset column based upon common column groupings. Akin to the SQL "group by" command (PR #123). + - `bedtools/bedtools_merge`: Merges overlapping BED/GFF/VCF entries into a single interval (PR #118). - `bedtools/bedtools_bamtofastq`: Convert BAM alignments to FASTQ files (PR #101). - `bedtools/bedtools_bedtobam`: Converts genomic feature records (bed/gff/vcf) to BAM format (PR #111). + - `bedtools/bedtools_links`: Creates an HTML file with links to an instance of the UCSC Genome Browser for all features / intervals in a (bed/gff/vcf) file (PR #137). * `qualimap/qualimap_rnaseq`: RNA-seq QC analysis using qualimap (PR #74). +* `rsem/rsem_prepare_reference`: Prepare transcript references for RSEM (PR #89). + + ## MINOR CHANGES * `busco` components: update BUSCO to `5.7.1` (PR #72). diff --git a/src/bedtools/bedtools_groupby/config.vsh.yaml b/src/bedtools/bedtools_groupby/config.vsh.yaml new file mode 100644 index 00000000..89c4845b --- /dev/null +++ b/src/bedtools/bedtools_groupby/config.vsh.yaml @@ -0,0 +1,155 @@ +name: bedtools_groupby +namespace: bedtools +description: | + Summarizes a dataset column based upon common column groupings. + Akin to the SQL "group by" command. +keywords: [groupby, BED] +links: + documentation: https://bedtools.readthedocs.io/en/latest/content/tools/groupby.html + repository: https://github.com/arq5x/bedtools2 + homepage: https://bedtools.readthedocs.io/en/latest/# + issue_tracker: https://github.com/arq5x/bedtools2/issues +references: + doi: 10.1093/bioinformatics/btq033 +license: MIT +requirements: + commands: [bedtools] +authors: + - __merge__: /src/_authors/theodoro_gasperin.yaml + roles: [ author, maintainer ] + +argument_groups: + - name: Inputs + arguments: + - name: --input + alternatives: -i + type: file + direction: input + description: | + The input BED file to be used. + required: true + example: input_a.bed + + - name: Outputs + arguments: + - name: --output + type: file + direction: output + description: | + The output groupby BED file. + required: true + example: output.bed + + - name: Options + arguments: + - name: --groupby + alternatives: [-g, -grp] + type: string + description: | + Specify the columns (1-based) for the grouping. + The columns must be comma separated. + - Default: 1,2,3 + required: true + + - name: --column + alternatives: [-c, -opCols] + type: integer + description: | + Specify the column (1-based) that should be summarized. + required: true + + - name: --operation + alternatives: [-o, -ops] + type: string + description: | + Specify the operation that should be applied to opCol. + Valid operations: + sum, count, count_distinct, min, max, + mean, median, mode, antimode, + stdev, sstdev (sample standard dev.), + collapse (i.e., print a comma separated list (duplicates allowed)), + distinct (i.e., print a comma separated list (NO duplicates allowed)), + distinct_sort_num (as distinct, but sorted numerically, ascending), + distinct_sort_num_desc (as distinct, but sorted numerically, descending), + concat (i.e., merge values into a single, non-delimited string), + freqdesc (i.e., print desc. list of values:freq) + freqasc (i.e., print asc. list of values:freq) + first (i.e., print first value) + last (i.e., print last value) + + Default value: sum + + If there is only column, but multiple operations, all operations will be + applied on that column. Likewise, if there is only one operation, but + multiple columns, that operation will be applied to all columns. + Otherwise, the number of columns must match the the number of operations, + and will be applied in respective order. + E.g., "-c 5,4,6 -o sum,mean,count" will give the sum of column 5, + the mean of column 4, and the count of column 6. + The order of output columns will match the ordering given in the command. + + - name: --full + type: boolean_true + description: | + Print all columns from input file. The first line in the group is used. + Default: print only grouped columns. + + - name: --inheader + type: boolean_true + description: | + Input file has a header line - the first line will be ignored. + + - name: --outheader + type: boolean_true + description: | + Print header line in the output, detailing the column names. + If the input file has headers (-inheader), the output file + will use the input's column names. + If the input file has no headers, the output file + will use "col_1", "col_2", etc. as the column names. + + - name: --header + type: boolean_true + description: same as '-inheader -outheader'. + + - name: --ignorecase + type: boolean_true + description: | + Group values regardless of upper/lower case. + + - name: --precision + alternatives: -prec + type: integer + description: | + Sets the decimal precision for output. + default: 5 + + - name: --delimiter + alternatives: -delim + type: string + description: | + Specify a custom delimiter for the collapse operations. + example: "|" + default: "," + +resources: + - type: bash_script + path: script.sh + +test_resources: + - type: bash_script + path: test.sh + +engines: + - type: docker + image: debian:stable-slim + setup: + - type: apt + packages: [bedtools, procps] + - type: docker + run: | + echo "bedtools: \"$(bedtools --version | sed -n 's/^bedtools //p')\"" > /var/software_versions.txt + +runners: + - type: executable + - type: nextflow diff --git a/src/bedtools/bedtools_groupby/help.txt b/src/bedtools/bedtools_groupby/help.txt new file mode 100644 index 00000000..a631b4b1 --- /dev/null +++ b/src/bedtools/bedtools_groupby/help.txt @@ -0,0 +1,93 @@ +```bash +bedtools groupby +``` + +Tool: bedtools groupby +Version: v2.30.0 +Summary: Summarizes a dataset column based upon + common column groupings. Akin to the SQL "group by" command. + +Usage: bedtools groupby -g [group_column(s)] -c [op_column(s)] -o [ops] + cat [FILE] | bedtools groupby -g [group_column(s)] -c [op_column(s)] -o [ops] + +Options: + -i Input file. Assumes "stdin" if omitted. + + -g -grp Specify the columns (1-based) for the grouping. + The columns must be comma separated. + - Default: 1,2,3 + + -c -opCols Specify the column (1-based) that should be summarized. + - Required. + + -o -ops Specify the operation that should be applied to opCol. + Valid operations: + sum, count, count_distinct, min, max, + mean, median, mode, antimode, + stdev, sstdev (sample standard dev.), + collapse (i.e., print a comma separated list (duplicates allowed)), + distinct (i.e., print a comma separated list (NO duplicates allowed)), + distinct_sort_num (as distinct, but sorted numerically, ascending), + distinct_sort_num_desc (as distinct, but sorted numerically, descending), + concat (i.e., merge values into a single, non-delimited string), + freqdesc (i.e., print desc. list of values:freq) + freqasc (i.e., print asc. list of values:freq) + first (i.e., print first value) + last (i.e., print last value) + - Default: sum + + If there is only column, but multiple operations, all operations will be + applied on that column. Likewise, if there is only one operation, but + multiple columns, that operation will be applied to all columns. + Otherwise, the number of columns must match the the number of operations, + and will be applied in respective order. + E.g., "-c 5,4,6 -o sum,mean,count" will give the sum of column 5, + the mean of column 4, and the count of column 6. + The order of output columns will match the ordering given in the command. + + + -full Print all columns from input file. The first line in the group is used. + Default: print only grouped columns. + + -inheader Input file has a header line - the first line will be ignored. + + -outheader Print header line in the output, detailing the column names. + If the input file has headers (-inheader), the output file + will use the input's column names. + If the input file has no headers, the output file + will use "col_1", "col_2", etc. as the column names. + + -header same as '-inheader -outheader' + + -ignorecase Group values regardless of upper/lower case. + + -prec Sets the decimal precision for output (Default: 5) + + -delim Specify a custom delimiter for the collapse operations. + - Example: -delim "|" + - Default: ",". + +Examples: + $ cat ex1.out + chr1 10 20 A chr1 15 25 B.1 1000 ATAT + chr1 10 20 A chr1 25 35 B.2 10000 CGCG + + $ groupBy -i ex1.out -g 1,2,3,4 -c 9 -o sum + chr1 10 20 A 11000 + + $ groupBy -i ex1.out -grp 1,2,3,4 -opCols 9,9 -ops sum,max + chr1 10 20 A 11000 10000 + + $ groupBy -i ex1.out -g 1,2,3,4 -c 8,9 -o collapse,mean + chr1 10 20 A B.1,B.2, 5500 + + $ cat ex1.out | groupBy -g 1,2,3,4 -c 8,9 -o collapse,mean + chr1 10 20 A B.1,B.2, 5500 + + $ cat ex1.out | groupBy -g 1,2,3,4 -c 10 -o concat + chr1 10 20 A ATATCGCG + +Notes: + (1) The input file/stream should be sorted/grouped by the -grp. columns + (2) If -i is unspecified, input is assumed to come from stdin. + diff --git a/src/bedtools/bedtools_groupby/script.sh b/src/bedtools/bedtools_groupby/script.sh new file mode 100644 index 00000000..b8a40cdc --- /dev/null +++ b/src/bedtools/bedtools_groupby/script.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +## VIASH START +## VIASH END + +# Exit on error +set -eo pipefail + +# Unset parameters +unset_if_false=( + par_full + par_inheader + par_outheader + par_header + par_ignorecase +) + +for par in ${unset_if_false[@]}; do + test_val="${!par}" + [[ "$test_val" == "false" ]] && unset $par +done + +bedtools groupby \ + ${par_full:+-full} \ + ${par_inheader:+-inheader} \ + ${par_outheader:+-outheader} \ + ${par_header:+-header} \ + ${par_ignorecase:+-ignorecase} \ + ${par_precision:+-prec "$par_precision"} \ + ${par_delimiter:+-delim "$par_delimiter"} \ + -i "$par_input" \ + -g "$par_groupby" \ + -c "$par_column" \ + ${par_operation:+-o "$par_operation"} \ + > "$par_output" + \ No newline at end of file diff --git a/src/bedtools/bedtools_groupby/test.sh b/src/bedtools/bedtools_groupby/test.sh new file mode 100644 index 00000000..ce99a1ec --- /dev/null +++ b/src/bedtools/bedtools_groupby/test.sh @@ -0,0 +1,198 @@ +#!/bin/bash + +# exit on error +set -eo pipefail + +## VIASH START +meta_executable="target/executable/bedtools/bedtools_groupby/bedtools_groupby" +meta_resources_dir="src/bedtools/bedtools_groupby" +## VIASH END + +############################################# +# helper functions +assert_file_exists() { + [ -f "$1" ] || { echo "File '$1' does not exist" && exit 1; } +} +assert_file_not_empty() { + [ -s "$1" ] || { echo "File '$1' is empty but shouldn't be" && exit 1; } +} +assert_file_contains() { + grep -q "$2" "$1" || { echo "File '$1' does not contain '$2'" && exit 1; } +} +assert_identical_content() { + diff -a "$2" "$1" \ + || (echo "Files are not identical!" && exit 1) +} +############################################# + +# Create directories for tests +echo "Creating Test Data..." +TMPDIR=$(mktemp -d "$meta_temp_dir/XXXXXX") +function clean_up { + [[ -d "$TMPDIR" ]] && rm -r "$TMPDIR" +} +trap clean_up EXIT + +# Create and populate example.bed +cat << EOF > $TMPDIR/example.bed +# Header +chr21 9719758 9729320 variant1 chr21 9719768 9721892 ALR/Alpha 1004 + +chr21 9719758 9729320 variant1 chr21 9721905 9725582 ALR/Alpha 1010 + +chr21 9719758 9729320 variant1 chr21 9725582 9725977 L1PA3 3288 + +chr21 9719758 9729320 variant1 chr21 9726021 9729309 ALR/Alpha 1051 + +chr21 9729310 9757478 variant2 chr21 9729320 9729809 L1PA3 3897 - +chr21 9729310 9757478 variant2 chr21 9729809 9730866 L1P1 8367 + +chr21 9729310 9757478 variant2 chr21 9730866 9734026 ALR/Alpha 1036 - +chr21 9729310 9757478 variant2 chr21 9734037 9757471 ALR/Alpha 1182 - +chr21 9795588 9796685 variant3 chr21 9795589 9795713 (GAATG)n 308 + +chr21 9795588 9796685 variant3 chr21 9795736 9795894 (GAATG)n 683 + +chr21 9795588 9796685 variant3 chr21 9795911 9796007 (GAATG)n 345 + +chr21 9795588 9796685 variant3 chr21 9796028 9796187 (GAATG)n 756 + +chr21 9795588 9796685 variant3 chr21 9796202 9796615 (GAATG)n 891 + +chr21 9795588 9796685 variant3 chr21 9796637 9796824 (GAATG)n 621 + +EOF + +# Create and populate expected output files for different tests +cat << EOF > $TMPDIR/expected.bed +chr21 9719758 9729320 6353 +chr21 9729310 9757478 14482 +chr21 9795588 9796685 3604 +EOF +cat << EOF > $TMPDIR/expected_max.bed +chr21 9719758 9729320 variant1 3288 +chr21 9729310 9757478 variant2 8367 +chr21 9795588 9796685 variant3 891 +EOF +cat << EOF > $TMPDIR/expected_full.bed +chr21 9719758 9729320 variant1 chr21 9719768 9721892 ALR/Alpha 1004 + 6353 +chr21 9729310 9757478 variant2 chr21 9729320 9729809 L1PA3 3897 - 14482 +chr21 9795588 9796685 variant3 chr21 9795589 9795713 (GAATG)n 308 + 3604 +EOF +cat << EOF > $TMPDIR/expected_delimited.bed +chr21 9719758 9729320 variant1 1004;1010;3288;1051 +chr21 9729310 9757478 variant2 3897;8367;1036;1182 +chr21 9795588 9796685 variant3 308;683;345;756;891;621 +EOF +cat << EOF > $TMPDIR/expected_precision.bed +chr21 9719758 9729320 variant1 1.6e+03 +chr21 9729310 9757478 variant2 3.6e+03 +chr21 9795588 9796685 variant3 6e+02 +EOF + +# Test 1: without operation option, default operation is sum +mkdir "$TMPDIR/test1" && pushd "$TMPDIR/test1" > /dev/null + +echo "> Run bedtools groupby on BED file" +"$meta_executable" \ + --input "../example.bed" \ + --groupby "1,2,3" \ + --column "9" \ + --output "output.bed" + +# checks +assert_file_exists "output.bed" +assert_file_not_empty "output.bed" +assert_identical_content "output.bed" "../expected.bed" +echo "- test1 succeeded -" + +popd > /dev/null + +# Test 2: with operation max option +mkdir "$TMPDIR/test2" && pushd "$TMPDIR/test2" > /dev/null + +echo "> Run bedtools groupby on BED file with max operation" +"$meta_executable" \ + --input "../example.bed" \ + --groupby "1-4" \ + --column "9" \ + --operation "max" \ + --output "output.bed" + +# checks +assert_file_exists "output.bed" +assert_file_not_empty "output.bed" +assert_identical_content "output.bed" "../expected_max.bed" +echo "- test2 succeeded -" + +popd > /dev/null + +# Test 3: full option +mkdir "$TMPDIR/test3" && pushd "$TMPDIR/test3" > /dev/null + +echo "> Run bedtools groupby on BED file with full option" +"$meta_executable" \ + --input "../example.bed" \ + --groupby "1-4" \ + --column "9" \ + --full \ + --output "output.bed" + +# checks +assert_file_exists "output.bed" +assert_file_not_empty "output.bed" +assert_identical_content "output.bed" "../expected_full.bed" +echo "- test3 succeeded -" + +popd > /dev/null + +# Test 4: header option +mkdir "$TMPDIR/test4" && pushd "$TMPDIR/test4" > /dev/null + +echo "> Run bedtools groupby on BED file with header option" +"$meta_executable" \ + --input "../example.bed" \ + --groupby "1-4" \ + --column "9" \ + --header \ + --output "output.bed" + +# checks +assert_file_exists "output.bed" +assert_file_not_empty "output.bed" +assert_file_contains "output.bed" "# Header" +echo "- test4 succeeded -" + +popd > /dev/null + +# Test 5: Delimiter and collapse +mkdir "$TMPDIR/test5" && pushd "$TMPDIR/test5" > /dev/null + +echo "> Run bedtools groupby on BED file with delimiter and collapse options" +"$meta_executable" \ + --input "../example.bed" \ + --groupby "1-4" \ + --column "9" \ + --operation "collapse" \ + --delimiter ";" \ + --output "output.bed" + +# checks +assert_file_exists "output.bed" +assert_file_not_empty "output.bed" +assert_identical_content "output.bed" "../expected_delimited.bed" +echo "- test5 succeeded -" + +popd > /dev/null + +# Test 6: precision option +mkdir "$TMPDIR/test6" && pushd "$TMPDIR/test6" > /dev/null + +echo "> Run bedtools groupby on BED file with precision option" +"$meta_executable" \ + --input "../example.bed" \ + --groupby "1-4" \ + --column "9" \ + --operation "mean" \ + --precision 2 \ + --output "output.bed" + +# checks +assert_file_exists "output.bed" +assert_file_not_empty "output.bed" +assert_identical_content "output.bed" "../expected_precision.bed" +echo "- test6 succeeded -" + +popd > /dev/null + +echo "---- All tests succeeded! ----" +exit 0 diff --git a/src/bedtools/bedtools_links/config.vsh.yaml b/src/bedtools/bedtools_links/config.vsh.yaml new file mode 100644 index 00000000..b4e43cd3 --- /dev/null +++ b/src/bedtools/bedtools_links/config.vsh.yaml @@ -0,0 +1,91 @@ +name: bedtools_links +namespace: bedtools +description: | + Creates an HTML file with links to an instance of the UCSC Genome Browser for all features / intervals in a file. + This is useful for cases when one wants to manually inspect through a large set of annotations or features. +keywords: [Links, BED, GFF, VCF] +links: + documentation: https://bedtools.readthedocs.io/en/latest/content/tools/links.html + repository: https://github.com/arq5x/bedtools2 + homepage: https://bedtools.readthedocs.io/en/latest/# + issue_tracker: https://github.com/arq5x/bedtools2/issues +references: + doi: 10.1093/bioinformatics/btq033 +license: MIT +requirements: + commands: [bedtools] +authors: + - __merge__: /src/_authors/theodoro_gasperin.yaml + roles: [ author, maintainer ] + +argument_groups: + - name: Inputs + arguments: + - name: --input + alternatives: -i + type: file + description: Input file (bed/gff/vcf). + required: true + + - name: Outputs + arguments: + - name: --output + alternatives: -o + type: file + direction: output + description: Output HTML file to be written. + + - name: Options + description: | + By default, the links created will point to human (hg18) UCSC browser. + If you have a local mirror, you can override this behavior by supplying + the -base, -org, and -db options. + + For example, if the URL of your local mirror for mouse MM9 is called: + http://mymirror.myuniversity.edu, then you would use the following: + --base_url http://mymirror.myuniversity.edu + --organism mouse + --database mm9 + arguments: + - name: --base_url + alternatives: -base + type: string + description: | + The “basename” for the UCSC browser. + default: http://genome.ucsc.edu + + - name: --organism + alternatives: -org + type: string + description: | + The organism (e.g. mouse, human). + default: human + + - name: --database + alternatives: -db + type: string + description: | + The genome build. + default: hg18 + +resources: + - type: bash_script + path: script.sh + +test_resources: + - type: bash_script + path: test.sh + +engines: + - type: docker + image: debian:stable-slim + setup: + - type: apt + packages: [bedtools, procps] + - type: docker + run: | + echo "bedtools: \"$(bedtools --version | sed -n 's/^bedtools //p')\"" > /var/software_versions.txt + +runners: + - type: executable + - type: nextflow diff --git a/src/bedtools/bedtools_links/help.txt b/src/bedtools/bedtools_links/help.txt new file mode 100644 index 00000000..d848d989 --- /dev/null +++ b/src/bedtools/bedtools_links/help.txt @@ -0,0 +1,25 @@ +``` +bedtools links -h +``` + +Tool: bedtools links (aka linksBed) +Version: v2.30.0 +Summary: Creates HTML links to an UCSC Genome Browser from a feature file. + +Usage: bedtools links [OPTIONS] -i > out.html + +Options: + -base The browser basename. Default: http://genome.ucsc.edu + -org The organism. Default: human + -db The build. Default: hg18 + +Example: + By default, the links created will point to human (hg18) UCSC browser. + If you have a local mirror, you can override this behavior by supplying + the -base, -org, and -db options. + + For example, if the URL of your local mirror for mouse MM9 is called: + http://mymirror.myuniversity.edu, then you would use the following: + -base http://mymirror.myuniversity.edu + -org mouse + -db mm9 diff --git a/src/bedtools/bedtools_links/script.sh b/src/bedtools/bedtools_links/script.sh new file mode 100644 index 00000000..b8ee9a56 --- /dev/null +++ b/src/bedtools/bedtools_links/script.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +## VIASH START +## VIASH END + +set -eo pipefail + +# Execute bedtools links +bedtools links \ + ${par_base_url:+-base "$par_base_url"} \ + ${par_organism:+-org "$par_organism"} \ + ${par_database:+-db "$par_database"} \ + -i "$par_input" \ + > "$par_output" diff --git a/src/bedtools/bedtools_links/test.sh b/src/bedtools/bedtools_links/test.sh new file mode 100644 index 00000000..d79cbd6c --- /dev/null +++ b/src/bedtools/bedtools_links/test.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +# exit on error +set -eo pipefail + +############################################# +# helper functions +assert_file_exists() { + [ -f "$1" ] || { echo "File '$1' does not exist" && exit 1; } +} +assert_file_not_empty() { + [ -s "$1" ] || { echo "File '$1' is empty but shouldn't be" && exit 1; } +} +assert_file_contains() { + grep -q "$2" "$1" || { echo "File '$1' does not contain '$2'" && exit 1; } +} +assert_identical_content() { + diff -a "$2" "$1" \ + || (echo "Files are not identical!" && exit 1) +} +############################################# + +# Create directories for tests +echo "Creating Test Data..." +TMPDIR=$(mktemp -d "$meta_temp_dir/XXXXXX") +function clean_up { + [[ -d "$TMPDIR" ]] && rm -r "$TMPDIR" +} +trap clean_up EXIT + +# Create test data +cat < "$TMPDIR/genes.bed" +chr21 9928613 10012791 uc002yip.1 0 - +chr21 9928613 10012791 uc002yiq.1 0 - +chr21 9928613 10012791 uc002yir.1 0 - +chr21 9928613 10012791 uc010gkv.1 0 - +chr21 9928613 10061300 uc002yis.1 0 - +chr21 10042683 10120796 uc002yit.1 0 - +chr21 10042683 10120808 uc002yiu.1 0 - +chr21 10079666 10120808 uc002yiv.1 0 - +chr21 10080031 10081687 uc002yiw.1 0 - +chr21 10081660 10120796 uc002yix.2 0 - +EOF + +# Test 1: Default Use +mkdir "$TMPDIR/test1" && pushd "$TMPDIR/test1" > /dev/null + +echo "> Run bedtools_links on BED file" +"$meta_executable" \ + --input "../genes.bed" \ + --output "genes.html" + +# checks +assert_file_exists "genes.html" +assert_file_not_empty "genes.html" +assert_file_contains "genes.html" "uc002yip.1" +echo "- test1 succeeded -" + +popd > /dev/null + +# Test 2: Base URL +mkdir "$TMPDIR/test2" && pushd "$TMPDIR/test2" > /dev/null + +echo "> Run bedtools_links with base option" +"$meta_executable" \ + --input "../genes.bed" \ + --output "genes.html" \ + --base_url "http://genome.ucsc.edu" + +# checks +assert_file_exists "genes.html" +assert_file_not_empty "genes.html" +assert_file_contains "genes.html" "uc002yip.1" +echo "- test2 succeeded -" + +popd > /dev/null + +# Test 3: Organism and Genome Database Build +mkdir "$TMPDIR/test3" && pushd "$TMPDIR/test3" > /dev/null + +echo "> Run bedtools_links with organism option and genome database build" +"$meta_executable" \ + --input "../genes.bed" \ + --output "genes.html" \ + --base_url "http://genome.ucsc.edu" \ + --organism "mouse" \ + --database "mm9" + +# checks +assert_file_exists "genes.html" +assert_file_not_empty "genes.html" +assert_file_contains "genes.html" "uc002yip.1" +echo "- test3 succeeded -" + +popd > /dev/null + +echo "---- All tests succeeded! ----" +exit 0 diff --git a/src/bedtools/bedtools_merge/config.vsh.yaml b/src/bedtools/bedtools_merge/config.vsh.yaml new file mode 100644 index 00000000..45e4a01d --- /dev/null +++ b/src/bedtools/bedtools_merge/config.vsh.yaml @@ -0,0 +1,160 @@ +name: bedtools_merge +namespace: bedtools +description: | + Merges overlapping BED/GFF/VCF entries into a single interval. +links: + documentation: https://bedtools.readthedocs.io/en/latest/content/tools/merge.html + repository: https://github.com/arq5x/bedtools2 + homepage: https://bedtools.readthedocs.io/en/latest/# + issue_tracker: https://github.com/arq5x/bedtools2/issues +references: + doi: 10.1093/bioinformatics/btq033 +license: MIT +requirements: + commands: [bedtools] +authors: + - __merge__: /src/_authors/theodoro_gasperin.yaml + roles: [ author, maintainer ] + +argument_groups: + - name: Inputs + arguments: + - name: --input + alternatives: -i + type: file + description: Input file (BED/GFF/VCF) to be merged. + required: true + + - name: Outputs + arguments: + - name: --output + type: file + direction: output + description: Output merged file BED to be written. + required: true + + - name: Options + arguments: + - name: --strand + alternatives: -s + type: boolean_true + description: | + Force strandedness. That is, only merge features + that are on the same strand. + - By default, merging is done without respect to strand. + + - name: --specific_strand + alternatives: -S + type: string + choices: ["+", "-"] + description: | + Force merge for one specific strand only. + Follow with + or - to force merge from only + the forward or reverse strand, respectively. + - By default, merging is done without respect to strand. + + - name: --distance + alternatives: -d + type: integer + description: | + Maximum distance between features allowed for features + to be merged. + - Def. 0. That is, overlapping & book-ended features are merged. + - (INTEGER) + - Note: negative values enforce the number of b.p. required for overlap. + + - name: --columns + alternatives: -c + type: integer + description: | + Specify columns from the B file to map onto intervals in A. + Default: 5. + Multiple columns can be specified in a comma-delimited list. + + - name: --operation + alternatives: -o + type: string + description: | + Specify the operation that should be applied to -c. + Valid operations: + sum, min, max, absmin, absmax, + mean, median, mode, antimode + stdev, sstdev + collapse (i.e., print a delimited list (duplicates allowed)), + distinct (i.e., print a delimited list (NO duplicates allowed)), + distinct_sort_num (as distinct, sorted numerically, ascending), + distinct_sort_num_desc (as distinct, sorted numerically, desscending), + distinct_only (delimited list of only unique values), + count + count_distinct (i.e., a count of the unique values in the column), + first (i.e., just the first value in the column), + last (i.e., just the last value in the column), + Default: sum + Multiple operations can be specified in a comma-delimited list. + + If there is only column, but multiple operations, all operations will be + applied on that column. Likewise, if there is only one operation, but + multiple columns, that operation will be applied to all columns. + Otherwise, the number of columns must match the the number of operations, + and will be applied in respective order. + E.g., "-c 5,4,6 -o sum,mean,count" will give the sum of column 5, + the mean of column 4, and the count of column 6. + The order of output columns will match the ordering given in the command. + + - name: --delimiter + alternatives: -delim + type: string + description: | + Specify a custom delimiter for the collapse operations. + example: "|" + default: "," + + - name: --precision + alternatives: -prec + type: integer + description: | + Sets the decimal precision for output (Default: 5). + + - name: --bed + type: boolean_true + description: | + If using BAM input, write output as BED. + + - name: --header + type: boolean_true + description: | + Print the header from the A file prior to results. + + - name: --no_buffer + alternatives: -nobuf + type: boolean_true + description: | + Disable buffered output. Using this option will cause each line + of output to be printed as it is generated, rather than saved + in a buffer. This will make printing large output files + noticeably slower, but can be useful in conjunction with + other software tools and scripts that need to process one + line of bedtools output at a time. + +resources: + - type: bash_script + path: script.sh + +test_resources: + - type: bash_script + path: test.sh + - path: test_data + +engines: + - type: docker + image: debian:stable-slim + setup: + - type: apt + packages: [bedtools, procps] + - type: docker + run: | + echo "bedtools: \"$(bedtools --version | sed -n 's/^bedtools //p')\"" > /var/software_versions.txt + +runners: + - type: executable + - type: nextflow \ No newline at end of file diff --git a/src/bedtools/bedtools_merge/help.txt b/src/bedtools/bedtools_merge/help.txt new file mode 100644 index 00000000..bc78fc67 --- /dev/null +++ b/src/bedtools/bedtools_merge/help.txt @@ -0,0 +1,85 @@ +```bash +bedtools merge +``` + +Tool: bedtools merge (aka mergeBed) +Version: v2.30.0 +Summary: Merges overlapping BED/GFF/VCF entries into a single interval. + +Usage: bedtools merge [OPTIONS] -i + +Options: + -s Force strandedness. That is, only merge features + that are on the same strand. + - By default, merging is done without respect to strand. + + -S Force merge for one specific strand only. + Follow with + or - to force merge from only + the forward or reverse strand, respectively. + - By default, merging is done without respect to strand. + + -d Maximum distance between features allowed for features + to be merged. + - Def. 0. That is, overlapping & book-ended features are merged. + - (INTEGER) + - Note: negative values enforce the number of b.p. required for overlap. + + -c Specify columns from the B file to map onto intervals in A. + Default: 5. + Multiple columns can be specified in a comma-delimited list. + + -o Specify the operation that should be applied to -c. + Valid operations: + sum, min, max, absmin, absmax, + mean, median, mode, antimode + stdev, sstdev + collapse (i.e., print a delimited list (duplicates allowed)), + distinct (i.e., print a delimited list (NO duplicates allowed)), + distinct_sort_num (as distinct, sorted numerically, ascending), + distinct_sort_num_desc (as distinct, sorted numerically, desscending), + distinct_only (delimited list of only unique values), + count + count_distinct (i.e., a count of the unique values in the column), + first (i.e., just the first value in the column), + last (i.e., just the last value in the column), + Default: sum + Multiple operations can be specified in a comma-delimited list. + + If there is only column, but multiple operations, all operations will be + applied on that column. Likewise, if there is only one operation, but + multiple columns, that operation will be applied to all columns. + Otherwise, the number of columns must match the the number of operations, + and will be applied in respective order. + E.g., "-c 5,4,6 -o sum,mean,count" will give the sum of column 5, + the mean of column 4, and the count of column 6. + The order of output columns will match the ordering given in the command. + + + -delim Specify a custom delimiter for the collapse operations. + - Example: -delim "|" + - Default: ",". + + -prec Sets the decimal precision for output (Default: 5) + + -bed If using BAM input, write output as BED. + + -header Print the header from the A file prior to results. + + -nobuf Disable buffered output. Using this option will cause each line + of output to be printed as it is generated, rather than saved + in a buffer. This will make printing large output files + noticeably slower, but can be useful in conjunction with + other software tools and scripts that need to process one + line of bedtools output at a time. + + -iobuf Specify amount of memory to use for input buffer. + Takes an integer argument. Optional suffixes K/M/G supported. + Note: currently has no effect with compressed files. + +Notes: + (1) The input file (-i) file must be sorted by chrom, then start. + + + + +***** ERROR: No input file given. Exiting. ***** diff --git a/src/bedtools/bedtools_merge/script.sh b/src/bedtools/bedtools_merge/script.sh new file mode 100644 index 00000000..db50dd83 --- /dev/null +++ b/src/bedtools/bedtools_merge/script.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +## VIASH START +## VIASH END + +# Exit on error +set -eo pipefail + +# Unset parameters +unset_if_false=( + par_strand + par_bed + par_header + par_no_buffer +) + +for par in ${unset_if_false[@]}; do + test_val="${!par}" + [[ "$test_val" == "false" ]] && unset $par +done + +# Execute bedtools merge with the provided arguments +bedtools merge \ + ${par_strand:+-s} \ + ${par_specific_strand:+-S "$par_specific_strand"} \ + ${par_bed:+-bed} \ + ${par_header:+-header} \ + ${par_no_buffer:+-nobuf} \ + ${par_distance:+-d "$par_distance"} \ + ${par_columns:+-c "$par_columns"} \ + ${par_operation:+-o "$par_operation"} \ + ${par_delimiter:+-delim "$par_delimiter"} \ + ${par_precision:+-prec "$par_precision"} \ + -i "$par_input" \ + > "$par_output" diff --git a/src/bedtools/bedtools_merge/test.sh b/src/bedtools/bedtools_merge/test.sh new file mode 100644 index 00000000..e2b46c15 --- /dev/null +++ b/src/bedtools/bedtools_merge/test.sh @@ -0,0 +1,222 @@ +#!/bin/bash + +# exit on error +set -eo pipefail + +## VIASH START +meta_executable="target/executable/bedtools/bedtools_sort/bedtools_merge" +meta_resources_dir="src/bedtools/bedtools_merge" +## VIASH END + +# directory of the bam file +test_data="$meta_resources_dir/test_data" + +############################################# +# helper functions +assert_file_exists() { + [ -f "$1" ] || { echo "File '$1' does not exist" && exit 1; } +} +assert_file_not_empty() { + [ -s "$1" ] || { echo "File '$1' is empty but shouldn't be" && exit 1; } +} +assert_file_contains() { + grep -q "$2" "$1" || { echo "File '$1' does not contain '$2'" && exit 1; } +} +assert_identical_content() { + diff -a "$2" "$1" \ + || (echo "Files are not identical!" && exit 1) +} +############################################# + +# Create directories for tests +echo "Creating Test Data..." +TMPDIR=$(mktemp -d "$meta_temp_dir/XXXXXX") +function clean_up { + [[ -d "$TMPDIR" ]] && rm -r "$TMPDIR" +} +trap clean_up EXIT + +# Create and populate example files +printf "chr1\t100\t200\nchr1\t150\t250\nchr1\t300\t400\n" > "$TMPDIR/featureA.bed" +printf "chr1\t100\t200\ta1\t1\t+\nchr1\t180\t250\ta2\t2\t+\nchr1\t250\t500\ta3\t3\t-\nchr1\t501\t1000\ta4\t4\t+\n" > "$TMPDIR/featureB.bed" +printf "chr1\t100\t200\ta1\t1.9\t+\nchr1\t180\t250\ta2\t2.5\t+\nchr1\t250\t500\ta3\t3.3\t-\nchr1\t501\t1000\ta4\t4\t+\n" > "$TMPDIR/feature_precision.bed" + +# Create and populate feature.gff file +printf "##gff-version 3\n" > "$TMPDIR/feature.gff" +printf "chr1\t.\tgene\t1000\t2000\t.\t+\t.\tID=gene1;Name=Gene1\n" >> "$TMPDIR/feature.gff" +printf "chr1\t.\texon\t1000\t1200\t.\t+\t.\tID=exon1;Parent=transcript1\n" >> "$TMPDIR/feature.gff" +printf "chr1\t.\tCDS\t1000\t1200\t.\t+\t0\tID=cds1;Parent=transcript1\n" >> "$TMPDIR/feature.gff" +printf "chr1\t.\tCDS\t1500\t1700\t.\t+\t2\tID=cds2;Parent=transcript1\n" >> "$TMPDIR/feature.gff" +printf "chr2\t.\texon\t1500\t1700\t.\t+\t.\tID=exon2;Parent=transcript1\n" >> "$TMPDIR/feature.gff" +printf "chr3\t.\tmRNA\t1000\t2000\t.\t+\t.\tID=transcript1;Parent=gene1\n" >> "$TMPDIR/feature.gff" + +# Create expected output files +printf "chr1\t100\t250\nchr1\t300\t400\n" > "$TMPDIR/expected.bed" +printf "chr1\t100\t250\nchr1\t250\t500\nchr1\t501\t1000\n" > "$TMPDIR/expected_strand.bed" +printf "chr1\t100\t250\nchr1\t501\t1000\n" > "$TMPDIR/expected_specific_strand.bed" +printf "chr1\t128\t228\nchr1\t428\t528\n" > "$TMPDIR/expected_bam.bed" +printf "chr1\t100\t400\n" > "$TMPDIR/expected_distance.bed" +printf "chr1\t100\t500\t2\t1\t3\nchr1\t501\t1000\t4\t4\t4\n" > "$TMPDIR/expected_operation.bed" +printf "chr1\t100\t500\ta1|a2|a3\nchr1\t501\t1000\ta4\n" > "$TMPDIR/expected_delim.bed" +printf "chr1\t100\t500\t2.567\nchr1\t501\t1000\t4\n" > "$TMPDIR/expected_precision.bed" +printf "##gff-version 3\nchr1\t999\t2000\nchr2\t1499\t1700\nchr3\t999\t2000\n" > "$TMPDIR/expected_header.bed" + +# Test 1: Default sort on BED file +mkdir "$TMPDIR/test1" && pushd "$TMPDIR/test1" > /dev/null + +echo "> Run bedtools_merge on BED file" +"$meta_executable" \ + --input "../featureA.bed" \ + --output "output.bed" + +# # checks +assert_file_exists "output.bed" +assert_file_not_empty "output.bed" +assert_identical_content "output.bed" "../expected.bed" +echo "- test1 succeeded -" + +popd > /dev/null + +# Test 2: strand option +mkdir "$TMPDIR/test2" && pushd "$TMPDIR/test2" > /dev/null + +echo "> Run bedtools_merge on BED file with strand option" +"$meta_executable" \ + --input "../featureB.bed" \ + --output "output.bed" \ + --strand + +# checks +assert_file_exists "output.bed" +assert_file_not_empty "output.bed" +assert_identical_content "output.bed" "../expected_strand.bed" +echo "- test2 succeeded -" + +popd > /dev/null + +# Test 3: specific strand option +mkdir "$TMPDIR/test3" && pushd "$TMPDIR/test3" > /dev/null + +echo "> Run bedtools_merge on BED file with specific strand option" +"$meta_executable" \ + --input "../featureB.bed" \ + --output "output.bed" \ + --specific_strand "+" + +# checks +assert_file_exists "output.bed" +assert_file_not_empty "output.bed" +assert_identical_content "output.bed" "../expected_specific_strand.bed" +echo "- test3 succeeded -" + +popd > /dev/null + +# Test 4: BED option +mkdir "$TMPDIR/test4" && pushd "$TMPDIR/test4" > /dev/null + +echo "> Run bedtools_merge on BAM file with BED option" +"$meta_executable" \ + --input "$test_data/feature.bam" \ + --output "output.bed" \ + --bed + +# checks +assert_file_exists "output.bed" +assert_file_not_empty "output.bed" +assert_identical_content "output.bed" "../expected_bam.bed" +echo "- test4 succeeded -" + +popd > /dev/null + +# Test 5: distance option +mkdir "$TMPDIR/test5" && pushd "$TMPDIR/test5" > /dev/null + +echo "> Run bedtools_merge on BED file with distance option" +"$meta_executable" \ + --input "../featureA.bed" \ + --output "output.bed" \ + --distance -5 + +# checks +assert_file_exists "output.bed" +assert_file_not_empty "output.bed" +assert_identical_content "output.bed" "../expected.bed" +echo "- test5 succeeded -" + +popd > /dev/null + +# Test 6: columns option & operation option +mkdir "$TMPDIR/test6" && pushd "$TMPDIR/test6" > /dev/null + +echo "> Run bedtools_merge on BED file with columns & operation options" +"$meta_executable" \ + --input "../featureB.bed" \ + --output "output.bed" \ + --columns 5 \ + --operation "mean,min,max" + +# checks +assert_file_exists "output.bed" +assert_file_not_empty "output.bed" +assert_identical_content "output.bed" "../expected_operation.bed" +echo "- test6 succeeded -" + +popd > /dev/null + +# Test 7: delimeter option +mkdir "$TMPDIR/test7" && pushd "$TMPDIR/test7" > /dev/null + +echo "> Run bedtools_merge on BED file with delimeter option" +"$meta_executable" \ + --input "../featureB.bed" \ + --output "output.bed" \ + --columns 4 \ + --operation "collapse" \ + --delimiter "|" + +# checks +assert_file_exists "output.bed" +assert_file_not_empty "output.bed" +assert_identical_content "output.bed" "../expected_delim.bed" +echo "- test7 succeeded -" + +popd > /dev/null + +# Test 8: precision option +mkdir "$TMPDIR/test8" && pushd "$TMPDIR/test8" > /dev/null + +echo "> Run bedtools_merge on BED file with precision option" +"$meta_executable" \ + --input "../feature_precision.bed" \ + --output "output.bed" \ + --columns 5 \ + --operation "mean" \ + --precision 4 + +# checks +assert_file_exists "output.bed" +assert_file_not_empty "output.bed" +assert_identical_content "output.bed" "../expected_precision.bed" +echo "- test8 succeeded -" + +popd > /dev/null + +# Test 9: header option +mkdir "$TMPDIR/test9" && pushd "$TMPDIR/test9" > /dev/null + +echo "> Run bedtools_merge on GFF file with header option" +"$meta_executable" \ + --input "../feature.gff" \ + --output "output.gff" \ + --header + +# checks +assert_file_exists "output.gff" +assert_file_not_empty "output.gff" +assert_identical_content "output.gff" "../expected_header.bed" +echo "- test9 succeeded -" + +popd > /dev/null + +echo "---- All tests succeeded! ----" +exit 0 diff --git a/src/bedtools/bedtools_merge/test_data/feature.bam b/src/bedtools/bedtools_merge/test_data/feature.bam new file mode 100644 index 00000000..3d56a631 Binary files /dev/null and b/src/bedtools/bedtools_merge/test_data/feature.bam differ diff --git a/src/rsem/rsem_prepare_reference/config.vsh.yaml b/src/rsem/rsem_prepare_reference/config.vsh.yaml new file mode 100644 index 00000000..44915a2f --- /dev/null +++ b/src/rsem/rsem_prepare_reference/config.vsh.yaml @@ -0,0 +1,196 @@ +name: rsem_prepare_reference +namespace: rsem +description: | + RSEM is a software package for estimating gene and isoform expression levels from RNA-Seq data. This component prepares transcript references for RSEM. +keywords: ["Transcriptome", "Index"] +links: + homepage: http://deweylab.github.io/RSEM + documentation: https://deweylab.github.io/RSEM/rsem-prepare-reference.html + repository: https://github.com/deweylab/RSEM +references: + doi: 10.1186/1471-2105-12-323 +license: GPL-3.0 +requirements: + commands: [ rsem-prepare-reference ] +authors: + - __merge__: /src/_authors/sai_nirmayi_yasa.yaml + roles: [ author, maintainer ] + +argument_groups: + - name: Inputs + arguments: + - name: --reference_fasta_files + type: file + description: | + Semi-colon separated list of Multi-FASTA formatted files OR a directory name. If a directory name is specified, RSEM will read all files with suffix ".fa" or ".fasta" in this directory. The files should contain either the sequences of transcripts or an entire genome, depending on whether the '--gtf' option is used. + required: true + multiple: true + example: read1.fasta + - name: --reference_name + type: string + description: | + The name of the reference used. RSEM will generate several reference-related files that are prefixed by this name. This name can contain path information (e.g. '/ref/mm9'). + required: true + example: /ref/mm9 + + - name: Outputs + arguments: + - name: --output + type: file + description: Directory containing reference files generated by RSEM. + required: true + direction: output + + - name: Other options + arguments: + - name: --gtf + type: file + description: Assume that 'reference_fasta_files' contains the sequence of a genome, and extract transcript reference sequences using the gene annotations specified in the GTF file. If this and '--gff3' options are not provided, RSEM will assume 'reference_fasta_files' contains the reference transcripts. In this case, RSEM assumes that name of each sequence in the Multi-FASTA files is its transcript_id. + example: annotations.gtf + - name: --gff3 + type: file + description: GFF3 annotation file. Converted to GTF format with the file name 'reference_name.gtf'. Please make sure that 'reference_name.gtf' does not exist. + example: annotations.gff + - name: --gff3_rna_patterns + type: string + description: List of transcript categories (separated by semi-colon). Only transcripts that match the string will be extracted. + multiple: true + example: mRNA;rRNA + - name: --gff3_genes_as_transcripts + type: boolean_true + description: This option is designed for untypical organisms, such as viruses, whose GFF3 files only contain genes. RSEM will assume each gene as a unique transcript when it converts the GFF3 file into GTF format. + - name: --trusted_sources + type: string + description: List of trusted sources (separated by semi-colon). Only transcripts coming from these sources will be extracted. If this option is off, all sources are accepted. + multiple: true + example: ENSEMBL;HAVANA + - name: --transcript_to_gene_map + type: file + description: | + Use information from this file to map from transcript (isoform) ids to gene ids. Each line of this file should be of the form: + gene_id transcript_id + with the two fields separated by a tab character. + If you are using a GTF file for the "UCSC Genes" gene set from the UCSC Genome Browser, then the "knownIsoforms.txt" file (obtained from the "Downloads" section of the UCSC Genome Browser site) is of this format. + If this option is off, then the mapping of isoforms to genes depends on whether the '--gtf' option is specified. If '--gtf' is specified, then RSEM uses the "gene_id" and "transcript_id" attributes in the GTF file. Otherwise, RSEM assumes that each sequence in the reference sequence files is a separate gene. + example: isoforms.txt + - name: --allele_to_gene_map + type: file + description: | + Use information from to provide gene_id and transcript_id information for each allele-specific transcript. Each line of should be of the form: + gene_id transcript_id allele_id + with the fields separated by a tab character. + This option is designed for quantifying allele-specific expression. It is only valid if '--gtf' option is not specified. allele_id should be the sequence names presented in the Multi-FASTA-formatted files. + - name: --polyA + type: boolean_true + description: Add poly(A) tails to the end of all reference isoforms. The length of poly(A) tail added is specified by '--polyA-length' option. STAR aligner users may not want to use this option. + - name: --polyA_length + type: integer + description: The length of the poly(A) tails to be added. + example: 125 + - name: --no_polyA_subset + type: file + description: Only meaningful if '--polyA' is specified. Do not add poly(A) tails to those transcripts listed in this file containing a list of transcript_ids. + example: transcript_ids.txt + - name: --bowtie + type: boolean_true + description: Build Bowtie indices. + - name: --bowtie2 + type: boolean_true + description: Build Bowtie 2 indices. + - name: --star + type: boolean_true + description: Build STAR indices. + - name: --star_sjdboverhang + type: integer + description: Length of the genomic sequence around annotated junction. It is only used for STAR to build splice junctions database and not needed for Bowtie or Bowtie2. It will be passed as the --sjdbOverhang option to STAR. According to STAR's manual, its ideal value is max(ReadLength)-1, e.g. for 2x101 paired-end reads, the ideal value is 101-1=100. In most cases, the default value of 100 will work as well as the ideal value. (Default is 100) + example: 100 + - name: --hisat2_hca + type: boolean_true + description: Build HISAT2 indices on the transcriptome according to Human Cell Atlas (HCA) SMART-Seq2 pipeline. + - name: --quiet + alternatives: -q + type: boolean_true + description: Suppress the output of logging information. + + - name: Prior-enhanced RSEM options + arguments: + - name: --prep_pRSEM + type: boolean_true + description: A Boolean indicating whether to prepare reference files for pRSEM, including building Bowtie indices for a genome and selecting training set isoforms. The index files will be used for aligning ChIP-seq reads in prior-enhanced RSEM and the training set isoforms will be used for learning prior. A path to Bowtie executables and a mappability file in bigWig format are required when this option is on. Currently, Bowtie2 is not supported for prior-enhanced RSEM. + - name: --mappability_bigwig_file + type: file + description: Full path to a whole-genome mappability file in bigWig format. This file is required for running prior-enhanced RSEM. It is used for selecting a training set of isoforms for prior-learning. This file can be either downloaded from UCSC Genome Browser or generated by GEM (Derrien et al., 2012, PLoS One). + +resources: + - type: bash_script + path: script.sh + +test_resources: + - type: bash_script + path: test.sh + +engines: +- type: docker + image: ubuntu:22.04 + setup: + - type: apt + packages: + - build-essential + - gcc + - g++ + - make + - wget + - zlib1g-dev + - unzip xxd + - perl + - r-base + - bowtie2 + - pip + - git + - type: python + packages: bowtie + - type: docker + env: + - STAR_VERSION=2.7.11b + - RSEM_VERSION=1.3.3 + - BOWTIE_VERSION=1.3.1 + - TZ=Europe/Brussels + run: | + ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone && \ + cd /tmp && \ + wget --no-check-certificate https://github.com/alexdobin/STAR/archive/refs/tags/${STAR_VERSION}.zip && \ + unzip ${STAR_VERSION}.zip && \ + cd STAR-${STAR_VERSION}/source && \ + make STARstatic CXXFLAGS_SIMD=-std=c++11 && \ + cp STAR /usr/local/bin && \ + cd /tmp && \ + wget --no-check-certificate https://github.com/deweylab/RSEM/archive/refs/tags/v${RSEM_VERSION}.zip && \ + unzip v${RSEM_VERSION}.zip && \ + cd RSEM-${RSEM_VERSION} && \ + make && \ + make install && \ + cd /tmp && \ + wget --no-check-certificate -O bowtie-${BOWTIE_VERSION}-linux-x86_64.zip https://sourceforge.net/projects/bowtie-bio/files/bowtie/${BOWTIE_VERSION}/bowtie-${BOWTIE_VERSION}-linux-x86_64.zip/download && \ + unzip bowtie-${BOWTIE_VERSION}-linux-x86_64.zip && \ + cp bowtie-${BOWTIE_VERSION}-linux-x86_64/bowtie* /usr/local/bin && \ + cd /tmp && \ + git clone https://github.com/DaehwanKimLab/hisat2.git /tmp/hisat2 && \ + cd /tmp/hisat2 && \ + make && \ + cp -r hisat2* /usr/local/bin && \ + cd && \ + rm -rf /tmp/STAR-${STAR_VERSION} /tmp/${STAR_VERSION}.zip /tmp/bowtie-${BOWTIE_VERSION}-linux-x86_64 /tmp/hisat2 && \ + apt-get --purge autoremove -y ${PACKAGES} && \ + apt-get clean + + - type: docker + run: | + echo "RSEM: `rsem-calculate-expression --version | sed -e 's/Current version: RSEM v//g'`" > /var/software_versions.txt && \ + echo "STAR: `STAR --version`" >> /var/software_versions.txt && \ + echo "bowtie2: `bowtie2 --version | grep -oP '\d+\.\d+\.\d+'`" >> /var/software_versions.txt && \ + echo "bowtie: `bowtie --version | grep -oP 'bowtie-align-s version \K\d+\.\d+\.\d+'`" >> /var/software_versions.txt && \ + echo "HISAT2: `hisat2 --version | grep -oP 'hisat2-align-s version \K\d+\.\d+\.\d+'`" >> /var/software_versions.txt + +runners: + - type: executable + - type: nextflow \ No newline at end of file diff --git a/src/rsem/rsem_prepare_reference/help.txt b/src/rsem/rsem_prepare_reference/help.txt new file mode 100644 index 00000000..c69899ec --- /dev/null +++ b/src/rsem/rsem_prepare_reference/help.txt @@ -0,0 +1,207 @@ +```bash +rsem-prepare-reference --help +``` + +NAME +rsem-prepare-reference - Prepare transcript references for RSEM and optionally build BOWTIE/BOWTIE2/STAR/HISAT2(transcriptome) indices. + +SYNOPSIS + rsem-prepare-reference [options] reference_fasta_file(s) reference_name +ARGUMENTS +reference_fasta_file(s) +Either a comma-separated list of Multi-FASTA formatted files OR a directory name. If a directory name is specified, RSEM will read all files with suffix ".fa" or ".fasta" in this directory. The files should contain either the sequences of transcripts or an entire genome, depending on whether the '--gtf' option is used. + +reference name +The name of the reference used. RSEM will generate several reference-related files that are prefixed by this name. This name can contain path information (e.g. '/ref/mm9'). + +OPTIONS +--gtf +If this option is on, RSEM assumes that 'reference_fasta_file(s)' contains the sequence of a genome, and will extract transcript reference sequences using the gene annotations specified in , which should be in GTF format. + +If this and '--gff3' options are off, RSEM will assume 'reference_fasta_file(s)' contains the reference transcripts. In this case, RSEM assumes that name of each sequence in the Multi-FASTA files is its transcript_id. + +(Default: off) + +--gff3 +The annotation file is in GFF3 format instead of GTF format. RSEM will first convert it to GTF format with the file name 'reference_name.gtf'. Please make sure that 'reference_name.gtf' does not exist. (Default: off) + +--gff3-RNA-patterns + is a comma-separated list of transcript categories, e.g. "mRNA,rRNA". Only transcripts that match the will be extracted. (Default: "mRNA") + +--gff3-genes-as-transcripts +This option is designed for untypical organisms, such as viruses, whose GFF3 files only contain genes. RSEM will assume each gene as a unique transcript when it converts the GFF3 file into GTF format. + +--trusted-sources + is a comma-separated list of trusted sources, e.g. "ENSEMBL,HAVANA". Only transcripts coming from these sources will be extracted. If this option is off, all sources are accepted. (Default: off) + +--transcript-to-gene-map +Use information from to map from transcript (isoform) ids to gene ids. Each line of should be of the form: + +gene_id transcript_id + +with the two fields separated by a tab character. + +If you are using a GTF file for the "UCSC Genes" gene set from the UCSC Genome Browser, then the "knownIsoforms.txt" file (obtained from the "Downloads" section of the UCSC Genome Browser site) is of this format. + +If this option is off, then the mapping of isoforms to genes depends on whether the '--gtf' option is specified. If '--gtf' is specified, then RSEM uses the "gene_id" and "transcript_id" attributes in the GTF file. Otherwise, RSEM assumes that each sequence in the reference sequence files is a separate gene. + +(Default: off) + +--allele-to-gene-map +Use information from to provide gene_id and transcript_id information for each allele-specific transcript. Each line of should be of the form: + +gene_id transcript_id allele_id + +with the fields separated by a tab character. + +This option is designed for quantifying allele-specific expression. It is only valid if '--gtf' option is not specified. allele_id should be the sequence names presented in the Multi-FASTA-formatted files. + +(Default: off) + +--polyA +Add poly(A) tails to the end of all reference isoforms. The length of poly(A) tail added is specified by '--polyA-length' option. STAR aligner users may not want to use this option. (Default: do not add poly(A) tail to any of the isoforms) + +--polyA-length +The length of the poly(A) tails to be added. (Default: 125) + +--no-polyA-subset +Only meaningful if '--polyA' is specified. Do not add poly(A) tails to those transcripts listed in . is a file containing a list of transcript_ids. (Default: off) + +--bowtie +Build Bowtie indices. (Default: off) + +--bowtie-path +The path to the Bowtie executables. (Default: the path to Bowtie executables is assumed to be in the user's PATH environment variable) + +--bowtie2 +Build Bowtie 2 indices. (Default: off) + +--bowtie2-path +The path to the Bowtie 2 executables. (Default: the path to Bowtie 2 executables is assumed to be in the user's PATH environment variable) + +--star +Build STAR indices. (Default: off) + +--star-path +The path to STAR's executable. (Default: the path to STAR executable is assumed to be in user's PATH environment variable) + +--star-sjdboverhang +Length of the genomic sequence around annotated junction. It is only used for STAR to build splice junctions database and not needed for Bowtie or Bowtie2. It will be passed as the --sjdbOverhang option to STAR. According to STAR's manual, its ideal value is max(ReadLength)-1, e.g. for 2x101 paired-end reads, the ideal value is 101-1=100. In most cases, the default value of 100 will work as well as the ideal value. (Default: 100) + +--hisat2-hca +Build HISAT2 indices on the transcriptome according to Human Cell Atlas (HCA) SMART-Seq2 pipeline. (Default: off) + +--hisat2-path +The path to the HISAT2 executables. (Default: the path to HISAT2 executables is assumed to be in the user's PATH environment variable) + +-p/--num-threads +Number of threads to use for building STAR's genome indices. (Default: 1) + +-q/--quiet +Suppress the output of logging information. (Default: off) + +-h/--help +Show help information. + +PRIOR-ENHANCED RSEM OPTIONS +--prep-pRSEM +A Boolean indicating whether to prepare reference files for pRSEM, including building Bowtie indices for a genome and selecting training set isoforms. The index files will be used for aligning ChIP-seq reads in prior-enhanced RSEM and the training set isoforms will be used for learning prior. A path to Bowtie executables and a mappability file in bigWig format are required when this option is on. Currently, Bowtie2 is not supported for prior-enhanced RSEM. (Default: off) + +--mappability-bigwig-file +Full path to a whole-genome mappability file in bigWig format. This file is required for running prior-enhanced RSEM. It is used for selecting a training set of isoforms for prior-learning. This file can be either downloaded from UCSC Genome Browser or generated by GEM (Derrien et al., 2012, PLoS One). (Default: "") + +DESCRIPTION +This program extracts/preprocesses the reference sequences for RSEM and prior-enhanced RSEM. It can optionally build Bowtie indices (with '--bowtie' option) and/or Bowtie 2 indices (with '--bowtie2' option) using their default parameters. It can also optionally build STAR indices (with '--star' option) using parameters from ENCODE3's STAR-RSEM pipeline. For prior-enhanced RSEM, it can build Bowtie genomic indices and select training set isoforms (with options '--prep-pRSEM' and '--mappability-bigwig-file '). If an alternative aligner is to be used, indices for that particular aligner can be built from either 'reference_name.idx.fa' or 'reference_name.n2g.idx.fa' (see OUTPUT for details). This program is used in conjunction with the 'rsem-calculate-expression' program. + +OUTPUT +This program will generate 'reference_name.grp', 'reference_name.ti', 'reference_name.transcripts.fa', 'reference_name.seq', 'reference_name.chrlist' (if '--gtf' is on), 'reference_name.idx.fa', 'reference_name.n2g.idx.fa', optional Bowtie/Bowtie 2 index files, and optional STAR index files. + +'reference_name.grp', 'reference_name.ti', 'reference_name.seq', and 'reference_name.chrlist' are used by RSEM internally. + +'reference_name.transcripts.fa' contains the extracted reference transcripts in Multi-FASTA format. Poly(A) tails are not added and it may contain lower case bases in its sequences if the corresponding genomic regions are soft-masked. + +'reference_name.idx.fa' and 'reference_name.n2g.idx.fa' are used by aligners to build their own indices. In these two files, all sequence bases are converted into upper case. In addition, poly(A) tails are added if '--polyA' option is set. The only difference between 'reference_name.idx.fa' and 'reference_name.n2g.idx.fa' is that 'reference_name.n2g.idx.fa' in addition converts all 'N' characters to 'G' characters. This conversion is in particular desired for aligners (e.g. Bowtie) that do not allow reads to overlap with 'N' characters in the reference sequences. Otherwise, 'reference_name.idx.fa' should be used to build the aligner's index files. RSEM uses 'reference_name.idx.fa' to build Bowtie 2 indices and 'reference_name.n2g.idx.fa' to build Bowtie indices. For visualizing the transcript-coordinate-based BAM files generated by RSEM in IGV, 'reference_name.idx.fa' should be imported as a "genome" (see Visualization section in README.md for details). + +If the whole genome is indexed for prior-enhanced RSEM, all the index files will be generated with prefix as 'reference_name_prsem'. Selected isoforms for training set are listed in the file 'reference_name_prsem.training_tr_crd' + +EXAMPLES +1) Suppose we have mouse RNA-Seq data and want to use the UCSC mm9 version of the mouse genome. We have downloaded the UCSC Genes transcript annotations in GTF format (as mm9.gtf) using the Table Browser and the knownIsoforms.txt file for mm9 from the UCSC Downloads. We also have all chromosome files for mm9 in the directory '/data/mm9'. We want to put the generated reference files under '/ref' with name 'mouse_0'. We do not add any poly(A) tails. Please note that GTF files generated from UCSC's Table Browser do not contain isoform-gene relationship information. For the UCSC Genes annotation, this information can be obtained from the knownIsoforms.txt file. Suppose we want to build Bowtie indices and Bowtie executables are found in '/sw/bowtie'. + +There are two ways to write the command: + + rsem-prepare-reference --gtf mm9.gtf \ + --transcript-to-gene-map knownIsoforms.txt \ + --bowtie \ + --bowtie-path /sw/bowtie \ + /data/mm9/chr1.fa,/data/mm9/chr2.fa,...,/data/mm9/chrM.fa \ + /ref/mouse_0 +OR + + rsem-prepare-reference --gtf mm9.gtf \ + --transcript-to-gene-map knownIsoforms.txt \ + --bowtie \ + --bowtie-path /sw/bowtie \ + /data/mm9 \ + /ref/mouse_0 +2) Suppose we also want to build Bowtie 2 indices in the above example and Bowtie 2 executables are found in '/sw/bowtie2', the command will be: + + rsem-prepare-reference --gtf mm9.gtf \ + --transcript-to-gene-map knownIsoforms.txt \ + --bowtie \ + --bowtie-path /sw/bowtie \ + --bowtie2 \ + --bowtie2-path /sw/bowtie2 \ + /data/mm9 \ + /ref/mouse_0 +3) Suppose we want to build STAR indices in the above example and save index files under '/ref' with name 'mouse_0'. Assuming STAR executable is '/sw/STAR', the command will be: + + rsem-prepare-reference --gtf mm9.gtf \ + --transcript-to-gene-map knownIsoforms.txt \ + --star \ + --star-path /sw/STAR \ + -p 8 \ + /data/mm9/chr1.fa,/data/mm9/chr2.fa,...,/data/mm9/chrM.fa \ + /ref/mouse_0 +OR + + rsem-prepare-reference --gtf mm9.gtf \ + --transcript-to-gene-map knownIsoforms.txt \ + --star \ + --star-path /sw/STAR \ + -p 8 \ + /data/mm9 + /ref/mouse_0 +STAR genome index files will be saved under '/ref/'. + +4) Suppose we want to prepare references for prior-enhanced RSEM in the above example. In this scenario, both STAR and Bowtie are required to build genomic indices - STAR for RNA-seq reads and Bowtie for ChIP-seq reads. Assuming their executables are under '/sw/STAR' and '/sw/Bowtie', respectively. Also, assuming the mappability file for mouse genome is '/data/mm9.bigWig'. The command will be: + + rsem-prepare-reference --gtf mm9.gtf \ + --transcript-to-gene-map knownIsoforms.txt \ + --star \ + --star-path /sw/STAR \ + -p 8 \ + --prep-pRSEM \ + --bowtie-path /sw/Bowtie \ + --mappability-bigwig-file /data/mm9.bigWig \ + /data/mm9/chr1.fa,/data/mm9/chr2.fa,...,/data/mm9/chrM.fa \ + /ref/mouse_0 +OR + + rsem-prepare-reference --gtf mm9.gtf \ + --transcript-to-gene-map knownIsoforms.txt \ + --star \ + --star-path /sw/STAR \ + -p 8 \ + --prep-pRSEM \ + --bowtie-path /sw/Bowtie \ + --mappability-bigwig-file /data/mm9.bigWig \ + /data/mm9 + /ref/mouse_0 +Both STAR and Bowtie's index files will be saved under '/ref/'. Bowtie files will have name prefix 'mouse_0_prsem' + +5) Suppose we only have transcripts from EST tags stored in 'mm9.fasta' and isoform-gene information stored in 'mapping.txt'. We want to add 125bp long poly(A) tails to all transcripts. The reference_name is set as 'mouse_125'. In addition, we do not want to build Bowtie/Bowtie 2 indices, and will use an alternative aligner to align reads against either 'mouse_125.idx.fa' or 'mouse_125.idx.n2g.fa': + + rsem-prepare-reference --transcript-to-gene-map mapping.txt \ + --polyA + mm9.fasta \ + mouse_125 \ No newline at end of file diff --git a/src/rsem/rsem_prepare_reference/script.sh b/src/rsem/rsem_prepare_reference/script.sh new file mode 100644 index 00000000..806804d8 --- /dev/null +++ b/src/rsem/rsem_prepare_reference/script.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +set -eo pipefail + +unset_if_false=( par_gff3_genes_as_transcripts par_polyA par_bowtie par_bowtie2 par_star par_hisat2_hca par_quiet par_prep_pRSEM ) + +for par in ${unset_if_false[@]}; do + test_val="${!par}" + [[ "$test_val" == "false" ]] && unset $par +done + +# replace ';' with ',' +par_reference_fasta_files=$(echo $par_reference_fasta_files | tr ';' ',') +par_gff3_rna_patterns=$(echo $par_gff3_rna_patterns | tr ';' ',') +par_trusted_sources=$(echo $par_trusted_sources | tr ';' ',') + +echo "$par_reference_fasta_files" +rsem-prepare-reference \ + ${par_gtf:+--gtf "${par_gtf}"} \ + ${par_gff3:+--gff3 "${par_gff3}"} \ + ${par_gff3_rna_patterns:+--gff3-RNA-patterns "${par_gff3_rna_patterns}"} \ + ${par_gff3_genes_as_transcripts:+--gff3-genes-as-transcripts "${par_gff3_genes_as_transcripts}"} \ + ${par_trusted_sources:+--trusted-sources "${par_trusted_sources}"} \ + ${par_transcript_to_gene_map:+--transcript-to-gene-map "${par_transcript_to_gene_map}"} \ + ${par_allele_to_gene_map:+--allele-to-gene-map "${par_allele_to_gene_map}"} \ + ${par_polyA:+--polyA} \ + ${par_polyA_length:+--polyA-length "${par_polyA_length}"} \ + ${par_no_polyA_subset:+--no-polyA-subset "${par_no_polyA_subset}"} \ + ${par_bowtie:+--bowtie} \ + ${par_bowtie2:+--bowtie2} \ + ${par_star:+--star} \ + ${par_star_sjdboverhang:+--star-sjdboverhang "${par_star_sjdboverhang}"} \ + ${par_hisat2_hca:+--hisat2-hca} \ + ${par_quiet:+--quiet} \ + ${par_prep_pRSEM:+--prep-pRSEM} \ + ${par_mappability_bigwig_file:+--mappability-bigwig-file "${par_mappability_bigwig_file}"} \ + ${meta_cpus:+--num-threads "${meta_cpus}"} \ + "${par_reference_fasta_files}" \ + "${par_reference_name}" + +mkdir -p "${par_output}" +mv ${par_reference_name}.* "${par_output}/" diff --git a/src/rsem/rsem_prepare_reference/test.sh b/src/rsem/rsem_prepare_reference/test.sh new file mode 100644 index 00000000..b38dd0a9 --- /dev/null +++ b/src/rsem/rsem_prepare_reference/test.sh @@ -0,0 +1,37 @@ + +#!/bin/bash + +set -e pipefail + +echo ">>> Testing $meta_functionality_name" + +cat > genome.fasta <<'EOF' +>Sheila +GCTAGCTCAGAAAAaaaNNN +EOF + +echo ">>> Prepare RSEM reference without gene annotations" +"$meta_executable" \ + --reference_fasta_files genome.fasta \ + --reference_name test \ + --output RSEM_index + +echo ">>> Checking whether output files exist" +[ ! -d "RSEM_index" ] && echo "RSEM index does not exist!" && exit 1 +[ ! -f "RSEM_index/test.grp" ] && echo "test.grp does not exist!" && exit 1 +[ ! -f "RSEM_index/test.n2g.idx.fa" ] && echo "test.n2g.idx.fa does not exist!" && exit 1 +[ ! -f "RSEM_index/test.ti" ] && echo "test.ti does not exist!" && exit 1 +[ ! -f "RSEM_index/test.idx.fa" ] && echo "test.idx.fa does not exist!" && exit 1 +[ ! -f "RSEM_index/test.seq" ] && echo "test.seq does not exist!" && exit 1 +[ ! -f "RSEM_index/test.transcripts.fa" ] && echo "test.transcripts.fa does not exist!" && exit 1 + +echo ">>> Checking whether output is correct" +[ ! -s "RSEM_index/test.grp" ] && echo "test.grp is empty!" && exit 1 +[ ! -s "RSEM_index/test.ti" ] && echo "test.ti is empty!" && exit 1 +[ ! -s "RSEM_index/test.seq" ] && echo "test.seq is empty!" && exit 1 +grep -q "GCTAGCTCAGAAAAaaaNNN" "RSEM_index/test.transcripts.fa" || { echo "The content of file 'test.transcripts.fa' seems to be incorrect." && exit 1; } +grep -q "GCTAGCTCAGAAAAAAANNN" "RSEM_index/test.idx.fa" || { echo "The content of file 'test.idx.fa' seems to be incorrect." && exit 1; } +grep -q "GCTAGCTCAGAAAAAAAGGG" "RSEM_index/test.n2g.idx.fa" || { echo "The content of file 'test.n2g.idx.fa' seems to be incorrect." && exit 1; } + +echo "All tests succeeded!" +exit 0 diff --git a/target/executable/agat/agat_convert_bed2gff/.config.vsh.yaml b/target/executable/agat/agat_convert_bed2gff/.config.vsh.yaml index e4835e2c..757ae7cb 100644 --- a/target/executable/agat/agat_convert_bed2gff/.config.vsh.yaml +++ b/target/executable/agat/agat_convert_bed2gff/.config.vsh.yaml @@ -234,15 +234,15 @@ build_info: engine: "docker|native" output: "target/executable/agat/agat_convert_bed2gff" executable: "target/executable/agat/agat_convert_bed2gff/agat_convert_bed2gff" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/agat/agat_convert_bed2gff/agat_convert_bed2gff b/target/executable/agat/agat_convert_bed2gff/agat_convert_bed2gff index 5ab1d981..a11fc845 100755 --- a/target/executable/agat/agat_convert_bed2gff/agat_convert_bed2gff +++ b/target/executable/agat/agat_convert_bed2gff/agat_convert_bed2gff @@ -2,7 +2,7 @@ # agat_convert_bed2gff main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -57,24 +57,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -255,9 +255,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -309,9 +309,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -331,7 +331,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -355,9 +356,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -514,9 +515,9 @@ RUN agat --version | sed 's/AGAT\s\(.*\)/agat: "\1"/' > /var/software_versions.t LABEL org.opencontainers.image.authors="Leïla Paquay" LABEL org.opencontainers.image.description="Companion container for running component agat agat_convert_bed2gff" -LABEL org.opencontainers.image.created="2024-08-21T11:36:01Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:49Z" LABEL org.opencontainers.image.source="https://github.com/NBISweden/AGAT" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -543,6 +544,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -573,14 +577,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -588,7 +595,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -596,7 +603,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -604,17 +613,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -771,6 +785,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -832,6 +858,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/agat/agat_convert_embl2gff/.config.vsh.yaml b/target/executable/agat/agat_convert_embl2gff/.config.vsh.yaml index 0bf07ed8..9c516a2c 100644 --- a/target/executable/agat/agat_convert_embl2gff/.config.vsh.yaml +++ b/target/executable/agat/agat_convert_embl2gff/.config.vsh.yaml @@ -224,15 +224,15 @@ build_info: engine: "docker|native" output: "target/executable/agat/agat_convert_embl2gff" executable: "target/executable/agat/agat_convert_embl2gff/agat_convert_embl2gff" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/agat/agat_convert_embl2gff/agat_convert_embl2gff b/target/executable/agat/agat_convert_embl2gff/agat_convert_embl2gff index 1504293c..b77c81be 100755 --- a/target/executable/agat/agat_convert_embl2gff/agat_convert_embl2gff +++ b/target/executable/agat/agat_convert_embl2gff/agat_convert_embl2gff @@ -2,7 +2,7 @@ # agat_convert_embl2gff main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -57,24 +57,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -245,9 +245,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -299,9 +299,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -321,7 +321,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -345,9 +346,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -504,9 +505,9 @@ RUN agat --version | sed 's/AGAT\s\(.*\)/agat: "\1"/' > /var/software_versions.t LABEL org.opencontainers.image.authors="Leïla Paquay" LABEL org.opencontainers.image.description="Companion container for running component agat agat_convert_embl2gff" -LABEL org.opencontainers.image.created="2024-08-21T11:36:00Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:49Z" LABEL org.opencontainers.image.source="https://github.com/NBISweden/AGAT" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -533,6 +534,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -563,14 +567,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -578,7 +585,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -586,7 +593,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -594,17 +603,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -778,6 +792,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -839,6 +865,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/agat/agat_convert_sp_gff2gtf/.config.vsh.yaml b/target/executable/agat/agat_convert_sp_gff2gtf/.config.vsh.yaml index 7e0482d1..4da1a29b 100644 --- a/target/executable/agat/agat_convert_sp_gff2gtf/.config.vsh.yaml +++ b/target/executable/agat/agat_convert_sp_gff2gtf/.config.vsh.yaml @@ -227,15 +227,15 @@ build_info: engine: "docker|native" output: "target/executable/agat/agat_convert_sp_gff2gtf" executable: "target/executable/agat/agat_convert_sp_gff2gtf/agat_convert_sp_gff2gtf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/agat/agat_convert_sp_gff2gtf/agat_convert_sp_gff2gtf b/target/executable/agat/agat_convert_sp_gff2gtf/agat_convert_sp_gff2gtf index 8174a50b..5e314cd5 100755 --- a/target/executable/agat/agat_convert_sp_gff2gtf/agat_convert_sp_gff2gtf +++ b/target/executable/agat/agat_convert_sp_gff2gtf/agat_convert_sp_gff2gtf @@ -2,7 +2,7 @@ # agat_convert_sp_gff2gtf main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -57,24 +57,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -259,9 +259,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -313,9 +313,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -335,7 +335,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -359,9 +360,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -518,9 +519,9 @@ RUN agat --version | sed 's/AGAT\s\(.*\)/agat: "\1"/' > /var/software_versions.t LABEL org.opencontainers.image.authors="Leïla Paquay" LABEL org.opencontainers.image.description="Companion container for running component agat agat_convert_sp_gff2gtf" -LABEL org.opencontainers.image.created="2024-08-21T11:36:01Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:50Z" LABEL org.opencontainers.image.source="https://github.com/NBISweden/AGAT" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -547,6 +548,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -577,14 +581,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -592,7 +599,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -600,7 +607,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -608,17 +617,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -749,6 +763,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -810,6 +836,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/agat/agat_convert_sp_gff2tsv/.config.vsh.yaml b/target/executable/agat/agat_convert_sp_gff2tsv/.config.vsh.yaml index 8b31b2a2..f22e197d 100644 --- a/target/executable/agat/agat_convert_sp_gff2tsv/.config.vsh.yaml +++ b/target/executable/agat/agat_convert_sp_gff2tsv/.config.vsh.yaml @@ -187,15 +187,15 @@ build_info: engine: "docker|native" output: "target/executable/agat/agat_convert_sp_gff2tsv" executable: "target/executable/agat/agat_convert_sp_gff2tsv/agat_convert_sp_gff2tsv" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/agat/agat_convert_sp_gff2tsv/agat_convert_sp_gff2tsv b/target/executable/agat/agat_convert_sp_gff2tsv/agat_convert_sp_gff2tsv index 4f562d66..93b6a60d 100755 --- a/target/executable/agat/agat_convert_sp_gff2tsv/agat_convert_sp_gff2tsv +++ b/target/executable/agat/agat_convert_sp_gff2tsv/agat_convert_sp_gff2tsv @@ -2,7 +2,7 @@ # agat_convert_sp_gff2tsv main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -57,24 +57,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -224,9 +224,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -278,9 +278,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -300,7 +300,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -324,9 +325,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -483,9 +484,9 @@ RUN agat --version | sed 's/AGAT\s\(.*\)/agat: "\1"/' > /var/software_versions.t LABEL org.opencontainers.image.authors="Leïla Paquay" LABEL org.opencontainers.image.description="Companion container for running component agat agat_convert_sp_gff2tsv" -LABEL org.opencontainers.image.created="2024-08-21T11:36:00Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:49Z" LABEL org.opencontainers.image.source="https://github.com/NBISweden/AGAT" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -512,6 +513,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -542,14 +546,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -557,7 +564,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -565,7 +572,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -573,17 +582,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -697,6 +711,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -758,6 +784,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/agat/agat_convert_sp_gxf2gxf/.config.vsh.yaml b/target/executable/agat/agat_convert_sp_gxf2gxf/.config.vsh.yaml index ba858099..3690c184 100644 --- a/target/executable/agat/agat_convert_sp_gxf2gxf/.config.vsh.yaml +++ b/target/executable/agat/agat_convert_sp_gxf2gxf/.config.vsh.yaml @@ -194,15 +194,15 @@ build_info: engine: "docker|native" output: "target/executable/agat/agat_convert_sp_gxf2gxf" executable: "target/executable/agat/agat_convert_sp_gxf2gxf/agat_convert_sp_gxf2gxf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/agat/agat_convert_sp_gxf2gxf/agat_convert_sp_gxf2gxf b/target/executable/agat/agat_convert_sp_gxf2gxf/agat_convert_sp_gxf2gxf index 9cdfad6f..e4b49754 100755 --- a/target/executable/agat/agat_convert_sp_gxf2gxf/agat_convert_sp_gxf2gxf +++ b/target/executable/agat/agat_convert_sp_gxf2gxf/agat_convert_sp_gxf2gxf @@ -2,7 +2,7 @@ # agat_convert_sp_gxf2gxf main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -57,24 +57,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -233,9 +233,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -287,9 +287,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -309,7 +309,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -333,9 +334,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -492,9 +493,9 @@ RUN agat --version | sed 's/AGAT\s\(.*\)/agat: "\1"/' > /var/software_versions.t LABEL org.opencontainers.image.authors="Leïla Paquay" LABEL org.opencontainers.image.description="Companion container for running component agat agat_convert_sp_gxf2gxf" -LABEL org.opencontainers.image.created="2024-08-21T11:36:00Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:48Z" LABEL org.opencontainers.image.source="https://github.com/NBISweden/AGAT" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -521,6 +522,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -551,14 +555,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -566,7 +573,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -574,7 +581,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -582,17 +591,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -706,6 +720,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -767,6 +793,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/arriba/.config.vsh.yaml b/target/executable/arriba/.config.vsh.yaml index e43ebec8..e5fd7374 100644 --- a/target/executable/arriba/.config.vsh.yaml +++ b/target/executable/arriba/.config.vsh.yaml @@ -705,15 +705,15 @@ build_info: engine: "docker|native" output: "target/executable/arriba" executable: "target/executable/arriba/arriba" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/arriba/arriba b/target/executable/arriba/arriba index 6a35626c..c8342c59 100755 --- a/target/executable/arriba/arriba +++ b/target/executable/arriba/arriba @@ -2,7 +2,7 @@ # arriba main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -57,24 +57,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -494,9 +494,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -548,9 +548,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -570,7 +570,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -594,9 +595,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -753,9 +754,9 @@ RUN arriba -h | grep 'Version:' 2>&1 | sed 's/Version:\s\(.*\)/arriba: "\1"/' > LABEL org.opencontainers.image.authors="Robrecht Cannoodt" LABEL org.opencontainers.image.description="Companion container for running component arriba" -LABEL org.opencontainers.image.created="2024-08-21T11:36:02Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:51Z" LABEL org.opencontainers.image.source="https://github.com/suhrig/arriba" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -782,6 +783,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -812,14 +816,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -827,7 +834,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -835,7 +842,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -843,17 +852,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -1539,6 +1553,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -1600,6 +1626,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/bcl_convert/.config.vsh.yaml b/target/executable/bcl_convert/.config.vsh.yaml index 56921f80..a5707636 100644 --- a/target/executable/bcl_convert/.config.vsh.yaml +++ b/target/executable/bcl_convert/.config.vsh.yaml @@ -417,15 +417,15 @@ build_info: engine: "docker|native" output: "target/executable/bcl_convert" executable: "target/executable/bcl_convert/bcl_convert" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/bcl_convert/bcl_convert b/target/executable/bcl_convert/bcl_convert index f033b570..d13b1ee3 100755 --- a/target/executable/bcl_convert/bcl_convert +++ b/target/executable/bcl_convert/bcl_convert @@ -2,7 +2,7 @@ # bcl_convert main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -58,24 +58,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -330,9 +330,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -384,9 +384,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -406,7 +406,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -430,9 +431,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -598,9 +599,9 @@ RUN echo "bcl-convert: \"$(bcl-convert -V 2>&1 >/dev/null | sed -n '/Version/ s/ LABEL org.opencontainers.image.authors="Toni Verbeiren, Dorien Roosen" LABEL org.opencontainers.image.description="Companion container for running component bcl_convert" -LABEL org.opencontainers.image.created="2024-08-21T11:36:02Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:50Z" LABEL org.opencontainers.image.source="https://github.com/viash-hub/biobox" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -627,6 +628,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -657,14 +661,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -672,7 +679,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -680,7 +687,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -688,17 +697,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -1020,6 +1034,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -1081,6 +1107,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/bd_rhapsody/bd_rhapsody_make_reference/.config.vsh.yaml b/target/executable/bd_rhapsody/bd_rhapsody_make_reference/.config.vsh.yaml index 3a99e885..bfe06fcf 100644 --- a/target/executable/bd_rhapsody/bd_rhapsody_make_reference/.config.vsh.yaml +++ b/target/executable/bd_rhapsody/bd_rhapsody_make_reference/.config.vsh.yaml @@ -269,15 +269,15 @@ build_info: engine: "docker|native" output: "target/executable/bd_rhapsody/bd_rhapsody_make_reference" executable: "target/executable/bd_rhapsody/bd_rhapsody_make_reference/bd_rhapsody_make_reference" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/bd_rhapsody/bd_rhapsody_make_reference/bd_rhapsody_make_reference b/target/executable/bd_rhapsody/bd_rhapsody_make_reference/bd_rhapsody_make_reference index f4aa929e..08a180ef 100755 --- a/target/executable/bd_rhapsody/bd_rhapsody_make_reference/bd_rhapsody_make_reference +++ b/target/executable/bd_rhapsody/bd_rhapsody_make_reference/bd_rhapsody_make_reference @@ -2,7 +2,7 @@ # bd_rhapsody_make_reference main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -58,24 +58,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -282,9 +282,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -336,9 +336,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -358,7 +358,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -382,9 +383,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -548,9 +549,9 @@ RUN echo "bdgenomics/rhapsody: 2.2.1" > /var/software_versions.txt LABEL org.opencontainers.image.authors="Robrecht Cannoodt, Weiwei Schultz" LABEL org.opencontainers.image.description="Companion container for running component bd_rhapsody bd_rhapsody_make_reference" -LABEL org.opencontainers.image.created="2024-08-21T11:36:05Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:53Z" LABEL org.opencontainers.image.source="https://bitbucket.org/CRSwDev/cwl/src/master/v2.2.1/Extra_Utilities/" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -577,6 +578,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -607,14 +611,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -622,7 +629,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -630,7 +637,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -638,17 +647,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -799,6 +813,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -860,6 +886,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/bedtools/bedtools_bamtofastq/.config.vsh.yaml b/target/executable/bedtools/bedtools_bamtofastq/.config.vsh.yaml index 320c4431..46157c2c 100644 --- a/target/executable/bedtools/bedtools_bamtofastq/.config.vsh.yaml +++ b/target/executable/bedtools/bedtools_bamtofastq/.config.vsh.yaml @@ -186,15 +186,15 @@ build_info: engine: "docker|native" output: "target/executable/bedtools/bedtools_bamtofastq" executable: "target/executable/bedtools/bedtools_bamtofastq/bedtools_bamtofastq" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/bedtools/bedtools_bamtofastq/bedtools_bamtofastq b/target/executable/bedtools/bedtools_bamtofastq/bedtools_bamtofastq index 5723f757..660cef5f 100755 --- a/target/executable/bedtools/bedtools_bamtofastq/bedtools_bamtofastq +++ b/target/executable/bedtools/bedtools_bamtofastq/bedtools_bamtofastq @@ -2,7 +2,7 @@ # bedtools_bamtofastq main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -57,24 +57,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -219,9 +219,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -273,9 +273,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -295,7 +295,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -319,9 +320,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -482,9 +483,9 @@ RUN echo "bedtools: \"$(bedtools --version | sed -n 's/^bedtools //p')\"" > /var LABEL org.opencontainers.image.authors="Theodoro Gasperin Terra Camargo" LABEL org.opencontainers.image.description="Companion container for running component bedtools bedtools_bamtofastq" -LABEL org.opencontainers.image.created="2024-08-21T11:35:51Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:39Z" LABEL org.opencontainers.image.source="https://github.com/arq5x/bedtools2" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -511,6 +512,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -541,14 +545,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -556,7 +563,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -564,7 +571,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -572,17 +581,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -689,6 +703,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -750,6 +776,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/bedtools/bedtools_bedtobam/.config.vsh.yaml b/target/executable/bedtools/bedtools_bedtobam/.config.vsh.yaml index 1ee26d8e..eab349cc 100644 --- a/target/executable/bedtools/bedtools_bedtobam/.config.vsh.yaml +++ b/target/executable/bedtools/bedtools_bedtobam/.config.vsh.yaml @@ -213,15 +213,15 @@ build_info: engine: "docker|native" output: "target/executable/bedtools/bedtools_bedtobam" executable: "target/executable/bedtools/bedtools_bedtobam/bedtools_bedtobam" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/bedtools/bedtools_bedtobam/bedtools_bedtobam b/target/executable/bedtools/bedtools_bedtobam/bedtools_bedtobam index a7f37938..674de14f 100755 --- a/target/executable/bedtools/bedtools_bedtobam/bedtools_bedtobam +++ b/target/executable/bedtools/bedtools_bedtobam/bedtools_bedtobam @@ -2,7 +2,7 @@ # bedtools_bedtobam main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -57,24 +57,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -232,9 +232,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -286,9 +286,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -308,7 +308,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -332,9 +333,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -495,9 +496,9 @@ RUN echo "bedtools: \"$(bedtools --version | sed -n 's/^bedtools //p')\"" > /var LABEL org.opencontainers.image.authors="Theodoro Gasperin Terra Camargo" LABEL org.opencontainers.image.description="Companion container for running component bedtools bedtools_bedtobam" -LABEL org.opencontainers.image.created="2024-08-21T11:35:51Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:40Z" LABEL org.opencontainers.image.source="https://github.com/arq5x/bedtools2" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -524,6 +525,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -554,14 +558,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -569,7 +576,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -577,7 +584,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -585,17 +594,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -729,6 +743,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -790,6 +816,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/bedtools/bedtools_getfasta/.config.vsh.yaml b/target/executable/bedtools/bedtools_getfasta/.config.vsh.yaml index 585d1e3b..acc09d12 100644 --- a/target/executable/bedtools/bedtools_getfasta/.config.vsh.yaml +++ b/target/executable/bedtools/bedtools_getfasta/.config.vsh.yaml @@ -231,15 +231,15 @@ build_info: engine: "docker|native" output: "target/executable/bedtools/bedtools_getfasta" executable: "target/executable/bedtools/bedtools_getfasta/bedtools_getfasta" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/bedtools/bedtools_getfasta/bedtools_getfasta b/target/executable/bedtools/bedtools_getfasta/bedtools_getfasta index cd873a08..29a79439 100755 --- a/target/executable/bedtools/bedtools_getfasta/bedtools_getfasta +++ b/target/executable/bedtools/bedtools_getfasta/bedtools_getfasta @@ -2,7 +2,7 @@ # bedtools_getfasta main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -57,24 +57,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -262,9 +262,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -316,9 +316,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -338,7 +338,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -362,9 +363,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -525,9 +526,9 @@ RUN echo "bedtools: \"$(bedtools --version | sed -n 's/^bedtools //p')\"" > /var LABEL org.opencontainers.image.authors="Dries Schaumont" LABEL org.opencontainers.image.description="Companion container for running component bedtools bedtools_getfasta" -LABEL org.opencontainers.image.created="2024-08-21T11:35:50Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:38Z" LABEL org.opencontainers.image.source="https://github.com/arq5x/bedtools2" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -554,6 +555,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -584,14 +588,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -599,7 +606,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -607,7 +614,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -615,17 +624,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -760,6 +774,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -821,6 +847,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/bedtools/bedtools_groupby/.config.vsh.yaml b/target/executable/bedtools/bedtools_groupby/.config.vsh.yaml new file mode 100644 index 00000000..240e82f6 --- /dev/null +++ b/target/executable/bedtools/bedtools_groupby/.config.vsh.yaml @@ -0,0 +1,299 @@ +name: "bedtools_groupby" +namespace: "bedtools" +version: "main" +authors: +- name: "Theodoro Gasperin Terra Camargo" + roles: + - "author" + - "maintainer" + info: + links: + email: "theodorogtc@gmail.com" + github: "tgaspe" + linkedin: "theodoro-gasperin-terra-camargo" + organizations: + - name: "Data Intuitive" + href: "https://www.data-intuitive.com" + role: "Bioinformatician" +argument_groups: +- name: "Inputs" + arguments: + - type: "file" + name: "--input" + alternatives: + - "-i" + description: "The input BED file to be used.\n" + info: null + example: + - "input_a.bed" + must_exist: true + create_parent: true + required: true + direction: "input" + multiple: false + multiple_sep: ";" +- name: "Outputs" + arguments: + - type: "file" + name: "--output" + description: "The output groupby BED file. \n" + info: null + example: + - "output.bed" + must_exist: true + create_parent: true + required: true + direction: "output" + multiple: false + multiple_sep: ";" +- name: "Options" + arguments: + - type: "string" + name: "--groupby" + alternatives: + - "-g" + - "-grp" + description: "Specify the columns (1-based) for the grouping.\nThe columns must\ + \ be comma separated.\n- Default: 1,2,3 \n" + info: null + required: true + direction: "input" + multiple: false + multiple_sep: ";" + - type: "integer" + name: "--column" + alternatives: + - "-c" + - "-opCols" + description: "Specify the column (1-based) that should be summarized.\n" + info: null + required: true + direction: "input" + multiple: false + multiple_sep: ";" + - type: "string" + name: "--operation" + alternatives: + - "-o" + - "-ops" + description: "Specify the operation that should be applied to opCol.\nValid operations:\n\ + \ sum, count, count_distinct, min, max,\n mean, median, mode, antimode,\n\ + \ stdev, sstdev (sample standard dev.),\n collapse (i.e., print a comma\ + \ separated list (duplicates allowed)), \n distinct (i.e., print a comma\ + \ separated list (NO duplicates allowed)), \n distinct_sort_num (as distinct,\ + \ but sorted numerically, ascending), \n distinct_sort_num_desc (as distinct,\ + \ but sorted numerically, descending), \n concat (i.e., merge values into\ + \ a single, non-delimited string), \n freqdesc (i.e., print desc. list of\ + \ values:freq)\n freqasc (i.e., print asc. list of values:freq)\n first\ + \ (i.e., print first value)\n last (i.e., print last value)\n\nDefault value:\ + \ sum \n\nIf there is only column, but multiple operations, all operations\ + \ will be\napplied on that column. Likewise, if there is only one operation,\ + \ but\nmultiple columns, that operation will be applied to all columns.\nOtherwise,\ + \ the number of columns must match the the number of operations,\nand will be\ + \ applied in respective order.\nE.g., \"-c 5,4,6 -o sum,mean,count\" will give\ + \ the sum of column 5,\nthe mean of column 4, and the count of column 6.\nThe\ + \ order of output columns will match the ordering given in the command.\n" + info: null + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "boolean_true" + name: "--full" + description: "Print all columns from input file. The first line in the group is\ + \ used.\nDefault: print only grouped columns.\n" + info: null + direction: "input" + - type: "boolean_true" + name: "--inheader" + description: "Input file has a header line - the first line will be ignored.\n" + info: null + direction: "input" + - type: "boolean_true" + name: "--outheader" + description: "Print header line in the output, detailing the column names. \n\ + If the input file has headers (-inheader), the output file\nwill use the input's\ + \ column names.\nIf the input file has no headers, the output file\nwill use\ + \ \"col_1\", \"col_2\", etc. as the column names.\n" + info: null + direction: "input" + - type: "boolean_true" + name: "--header" + description: "same as '-inheader -outheader'." + info: null + direction: "input" + - type: "boolean_true" + name: "--ignorecase" + description: "Group values regardless of upper/lower case.\n" + info: null + direction: "input" + - type: "integer" + name: "--precision" + alternatives: + - "-prec" + description: "Sets the decimal precision for output. \n" + info: null + default: + - 5 + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "string" + name: "--delimiter" + alternatives: + - "-delim" + description: "Specify a custom delimiter for the collapse operations.\n" + info: null + example: + - "|" + default: + - "," + required: false + direction: "input" + multiple: false + multiple_sep: ";" +resources: +- type: "bash_script" + path: "script.sh" + is_executable: true +description: "Summarizes a dataset column based upon common column groupings. \nAkin\ + \ to the SQL \"group by\" command.\n" +test_resources: +- type: "bash_script" + path: "test.sh" + is_executable: true +info: null +status: "enabled" +requirements: + commands: + - "ps" +keywords: +- "groupby" +- "BED" +license: "MIT" +references: + doi: + - "10.1093/bioinformatics/btq033" +links: + repository: "https://github.com/arq5x/bedtools2" + homepage: "https://bedtools.readthedocs.io/en/latest/#" + documentation: "https://bedtools.readthedocs.io/en/latest/content/tools/groupby.html" + issue_tracker: "https://github.com/arq5x/bedtools2/issues" +runners: +- type: "executable" + id: "executable" + docker_setup_strategy: "ifneedbepullelsecachedbuild" +- type: "nextflow" + id: "nextflow" + directives: + tag: "$id" + auto: + simplifyInput: true + simplifyOutput: false + transcript: false + publish: false + config: + labels: + mem1gb: "memory = 1000000000.B" + mem2gb: "memory = 2000000000.B" + mem5gb: "memory = 5000000000.B" + mem10gb: "memory = 10000000000.B" + mem20gb: "memory = 20000000000.B" + mem50gb: "memory = 50000000000.B" + mem100gb: "memory = 100000000000.B" + mem200gb: "memory = 200000000000.B" + mem500gb: "memory = 500000000000.B" + mem1tb: "memory = 1000000000000.B" + mem2tb: "memory = 2000000000000.B" + mem5tb: "memory = 5000000000000.B" + mem10tb: "memory = 10000000000000.B" + mem20tb: "memory = 20000000000000.B" + mem50tb: "memory = 50000000000000.B" + mem100tb: "memory = 100000000000000.B" + mem200tb: "memory = 200000000000000.B" + mem500tb: "memory = 500000000000000.B" + mem1gib: "memory = 1073741824.B" + mem2gib: "memory = 2147483648.B" + mem4gib: "memory = 4294967296.B" + mem8gib: "memory = 8589934592.B" + mem16gib: "memory = 17179869184.B" + mem32gib: "memory = 34359738368.B" + mem64gib: "memory = 68719476736.B" + mem128gib: "memory = 137438953472.B" + mem256gib: "memory = 274877906944.B" + mem512gib: "memory = 549755813888.B" + mem1tib: "memory = 1099511627776.B" + mem2tib: "memory = 2199023255552.B" + mem4tib: "memory = 4398046511104.B" + mem8tib: "memory = 8796093022208.B" + mem16tib: "memory = 17592186044416.B" + mem32tib: "memory = 35184372088832.B" + mem64tib: "memory = 70368744177664.B" + mem128tib: "memory = 140737488355328.B" + mem256tib: "memory = 281474976710656.B" + mem512tib: "memory = 562949953421312.B" + cpu1: "cpus = 1" + cpu2: "cpus = 2" + cpu5: "cpus = 5" + cpu10: "cpus = 10" + cpu20: "cpus = 20" + cpu50: "cpus = 50" + cpu100: "cpus = 100" + cpu200: "cpus = 200" + cpu500: "cpus = 500" + cpu1000: "cpus = 1000" + debug: false + container: "docker" +engines: +- type: "docker" + id: "docker" + image: "debian:stable-slim" + target_registry: "images.viash-hub.com" + target_tag: "main" + namespace_separator: "/" + setup: + - type: "apt" + packages: + - "bedtools" + - "procps" + interactive: false + - type: "docker" + run: + - "echo \"bedtools: \\\"$(bedtools --version | sed -n 's/^bedtools //p')\\\"\"\ + \ > /var/software_versions.txt\n" + entrypoint: [] + cmd: null +- type: "native" + id: "native" +build_info: + config: "src/bedtools/bedtools_groupby/config.vsh.yaml" + runner: "executable" + engine: "docker|native" + output: "target/executable/bedtools/bedtools_groupby" + executable: "target/executable/bedtools/bedtools_groupby/bedtools_groupby" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" + git_remote: "https://github.com/viash-hub/biobox" +package_config: + name: "biobox" + version: "main" + description: "A collection of bioinformatics tools for working with sequence data.\n" + info: null + viash_version: "0.9.0-RC7" + source: "src" + target: "target" + config_mods: + - ".requirements.commands := ['ps']\n" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'main'" + keywords: + - "bioinformatics" + - "modules" + - "sequencing" + license: "MIT" + organization: "vsh" + links: + repository: "https://github.com/viash-hub/biobox" + issue_tracker: "https://github.com/viash-hub/biobox/issues" diff --git a/target/executable/bedtools/bedtools_groupby/bedtools_groupby b/target/executable/bedtools/bedtools_groupby/bedtools_groupby new file mode 100755 index 00000000..686b31d9 --- /dev/null +++ b/target/executable/bedtools/bedtools_groupby/bedtools_groupby @@ -0,0 +1,1410 @@ +#!/usr/bin/env bash + +# bedtools_groupby main +# +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a +# derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from +# Data Intuitive. +# +# The component may contain files which fall under a different license. The +# authors of this component should specify the license in the header of such +# files, or include a separate license file detailing the licenses of all included +# files. +# +# Component authors: +# * Theodoro Gasperin Terra Camargo (author, maintainer) + +set -e + +if [ -z "$VIASH_TEMP" ]; then + VIASH_TEMP=${VIASH_TEMP:-$VIASH_TMPDIR} + VIASH_TEMP=${VIASH_TEMP:-$VIASH_TEMPDIR} + VIASH_TEMP=${VIASH_TEMP:-$VIASH_TMP} + VIASH_TEMP=${VIASH_TEMP:-$TMPDIR} + VIASH_TEMP=${VIASH_TEMP:-$TMP} + VIASH_TEMP=${VIASH_TEMP:-$TEMPDIR} + VIASH_TEMP=${VIASH_TEMP:-$TEMP} + VIASH_TEMP=${VIASH_TEMP:-/tmp} +fi + +# define helper functions +# ViashQuote: put quotes around non flag values +# $1 : unquoted string +# return : possibly quoted string +# examples: +# ViashQuote --foo # returns --foo +# ViashQuote bar # returns 'bar' +# Viashquote --foo=bar # returns --foo='bar' +function ViashQuote { + if [[ "$1" =~ ^-+[a-zA-Z0-9_\-]+=.+$ ]]; then + echo "$1" | sed "s#=\(.*\)#='\1'#" + elif [[ "$1" =~ ^-+[a-zA-Z0-9_\-]+$ ]]; then + echo "$1" + else + echo "'$1'" + fi +} +# ViashRemoveFlags: Remove leading flag +# $1 : string with a possible leading flag +# return : string without possible leading flag +# examples: +# ViashRemoveFlags --foo=bar # returns bar +function ViashRemoveFlags { + echo "$1" | sed 's/^--*[a-zA-Z0-9_\-]*=//' +} +# ViashSourceDir: return the path of a bash file, following symlinks +# usage : ViashSourceDir ${BASH_SOURCE[0]} +# $1 : Should always be set to ${BASH_SOURCE[0]} +# returns : The absolute path of the bash file +function ViashSourceDir { + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" + done + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd +} +# ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks +# usage : ViashFindTargetDir 'ScriptPath' +# $1 : The location from where to start the upward search +# returns : The absolute path of the '.build.yaml' file +function ViashFindTargetDir { + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} + done + echo $source +} +# see https://en.wikipedia.org/wiki/Syslog#Severity_level +VIASH_LOGCODE_EMERGENCY=0 +VIASH_LOGCODE_ALERT=1 +VIASH_LOGCODE_CRITICAL=2 +VIASH_LOGCODE_ERROR=3 +VIASH_LOGCODE_WARNING=4 +VIASH_LOGCODE_NOTICE=5 +VIASH_LOGCODE_INFO=6 +VIASH_LOGCODE_DEBUG=7 +VIASH_VERBOSITY=$VIASH_LOGCODE_NOTICE + +# ViashLog: Log events depending on the verbosity level +# usage: ViashLog 1 alert Oh no something went wrong! +# $1: required verbosity level +# $2: display tag +# $3+: messages to display +# stdout: Your input, prepended by '[$2] '. +function ViashLog { + local required_level="$1" + local display_tag="$2" + shift 2 + if [ $VIASH_VERBOSITY -ge $required_level ]; then + >&2 echo "[$display_tag]" "$@" + fi +} + +# ViashEmergency: log events when the system is unstable +# usage: ViashEmergency Oh no something went wrong. +# stdout: Your input, prepended by '[emergency] '. +function ViashEmergency { + ViashLog $VIASH_LOGCODE_EMERGENCY emergency "$@" +} + +# ViashAlert: log events when actions must be taken immediately (e.g. corrupted system database) +# usage: ViashAlert Oh no something went wrong. +# stdout: Your input, prepended by '[alert] '. +function ViashAlert { + ViashLog $VIASH_LOGCODE_ALERT alert "$@" +} + +# ViashCritical: log events when a critical condition occurs +# usage: ViashCritical Oh no something went wrong. +# stdout: Your input, prepended by '[critical] '. +function ViashCritical { + ViashLog $VIASH_LOGCODE_CRITICAL critical "$@" +} + +# ViashError: log events when an error condition occurs +# usage: ViashError Oh no something went wrong. +# stdout: Your input, prepended by '[error] '. +function ViashError { + ViashLog $VIASH_LOGCODE_ERROR error "$@" +} + +# ViashWarning: log potentially abnormal events +# usage: ViashWarning Something may have gone wrong. +# stdout: Your input, prepended by '[warning] '. +function ViashWarning { + ViashLog $VIASH_LOGCODE_WARNING warning "$@" +} + +# ViashNotice: log significant but normal events +# usage: ViashNotice This just happened. +# stdout: Your input, prepended by '[notice] '. +function ViashNotice { + ViashLog $VIASH_LOGCODE_NOTICE notice "$@" +} + +# ViashInfo: log normal events +# usage: ViashInfo This just happened. +# stdout: Your input, prepended by '[info] '. +function ViashInfo { + ViashLog $VIASH_LOGCODE_INFO info "$@" +} + +# ViashDebug: log all events, for debugging purposes +# usage: ViashDebug This just happened. +# stdout: Your input, prepended by '[debug] '. +function ViashDebug { + ViashLog $VIASH_LOGCODE_DEBUG debug "$@" +} + +# find source folder of this component +VIASH_META_RESOURCES_DIR=`ViashSourceDir ${BASH_SOURCE[0]}` + +# find the root of the built components & dependencies +VIASH_TARGET_DIR=`ViashFindTargetDir $VIASH_META_RESOURCES_DIR` + +# define meta fields +VIASH_META_NAME="bedtools_groupby" +VIASH_META_FUNCTIONALITY_NAME="bedtools_groupby" +VIASH_META_EXECUTABLE="$VIASH_META_RESOURCES_DIR/$VIASH_META_NAME" +VIASH_META_CONFIG="$VIASH_META_RESOURCES_DIR/.config.vsh.yaml" +VIASH_META_TEMP_DIR="$VIASH_TEMP" + + +# ViashHelp: Display helpful explanation about this executable +function ViashHelp { + echo "bedtools_groupby main" + echo "" + echo "Summarizes a dataset column based upon common column groupings." + echo "Akin to the SQL \"group by\" command." + echo "" + echo "Inputs:" + echo " -i, --input" + echo " type: file, required parameter, file must exist" + echo " example: input_a.bed" + echo " The input BED file to be used." + echo "" + echo "Outputs:" + echo " --output" + echo " type: file, required parameter, output, file must exist" + echo " example: output.bed" + echo " The output groupby BED file." + echo "" + echo "Options:" + echo " -g, -grp, --groupby" + echo " type: string, required parameter" + echo " Specify the columns (1-based) for the grouping." + echo " The columns must be comma separated." + echo " - Default: 1,2,3" + echo "" + echo " -c, -opCols, --column" + echo " type: integer, required parameter" + echo " Specify the column (1-based) that should be summarized." + echo "" + echo " -o, -ops, --operation" + echo " type: string" + echo " Specify the operation that should be applied to opCol." + echo " Valid operations:" + echo " sum, count, count_distinct, min, max," + echo " mean, median, mode, antimode," + echo " stdev, sstdev (sample standard dev.)," + echo " collapse (i.e., print a comma separated list (duplicates allowed))," + echo " distinct (i.e., print a comma separated list (NO duplicates" + echo " allowed))," + echo " distinct_sort_num (as distinct, but sorted numerically, ascending)," + echo " distinct_sort_num_desc (as distinct, but sorted numerically," + echo " descending)," + echo " concat (i.e., merge values into a single, non-delimited string)," + echo " freqdesc (i.e., print desc. list of values:freq)" + echo " freqasc (i.e., print asc. list of values:freq)" + echo " first (i.e., print first value)" + echo " last (i.e., print last value)" + echo " Default value: sum" + echo " If there is only column, but multiple operations, all operations will be" + echo " applied on that column. Likewise, if there is only one operation, but" + echo " multiple columns, that operation will be applied to all columns." + echo " Otherwise, the number of columns must match the the number of" + echo " operations," + echo " and will be applied in respective order." + echo " E.g., \"-c 5,4,6 -o sum,mean,count\" will give the sum of column 5," + echo " the mean of column 4, and the count of column 6." + echo " The order of output columns will match the ordering given in the" + echo " command." + echo "" + echo " --full" + echo " type: boolean_true" + echo " Print all columns from input file. The first line in the group is used." + echo " Default: print only grouped columns." + echo "" + echo " --inheader" + echo " type: boolean_true" + echo " Input file has a header line - the first line will be ignored." + echo "" + echo " --outheader" + echo " type: boolean_true" + echo " Print header line in the output, detailing the column names." + echo " If the input file has headers (-inheader), the output file" + echo " will use the input's column names." + echo " If the input file has no headers, the output file" + echo " will use \"col_1\", \"col_2\", etc. as the column names." + echo "" + echo " --header" + echo " type: boolean_true" + echo " same as '-inheader -outheader'." + echo "" + echo " --ignorecase" + echo " type: boolean_true" + echo " Group values regardless of upper/lower case." + echo "" + echo " -prec, --precision" + echo " type: integer" + echo " default: 5" + echo " Sets the decimal precision for output." + echo "" + echo " -delim, --delimiter" + echo " type: string" + echo " default: ," + echo " example: |" + echo " Specify a custom delimiter for the collapse operations." +} + +# initialise variables +VIASH_MODE='run' +VIASH_ENGINE_ID='docker' + +######## Helper functions for setting up Docker images for viash ######## +# expects: ViashDockerBuild + +# ViashDockerInstallationCheck: check whether Docker is installed correctly +# +# examples: +# ViashDockerInstallationCheck +function ViashDockerInstallationCheck { + ViashDebug "Checking whether Docker is installed" + if [ ! command -v docker &> /dev/null ]; then + ViashCritical "Docker doesn't seem to be installed. See 'https://docs.docker.com/get-docker/' for instructions." + exit 1 + fi + + ViashDebug "Checking whether the Docker daemon is running" + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? + [[ $save =~ e ]] && set -e + if [ $out -ne 0 ]; then + ViashCritical "Docker daemon does not seem to be running. Try one of the following:" + ViashCritical "- Try running 'dockerd' in the command line" + ViashCritical "- See https://docs.docker.com/config/daemon/" + exit 1 + fi +} + +# ViashDockerRemoteTagCheck: check whether a Docker image is available +# on a remote. Assumes `docker login` has been performed, if relevant. +# +# $1 : image identifier with format `[registry/]image[:tag]` +# exit code $? : whether or not the image was found +# examples: +# ViashDockerRemoteTagCheck python:latest +# echo $? # returns '0' +# ViashDockerRemoteTagCheck sdaizudceahifu +# echo $? # returns '1' +function ViashDockerRemoteTagCheck { + docker manifest inspect $1 > /dev/null 2> /dev/null +} + +# ViashDockerLocalTagCheck: check whether a Docker image is available locally +# +# $1 : image identifier with format `[registry/]image[:tag]` +# exit code $? : whether or not the image was found +# examples: +# docker pull python:latest +# ViashDockerLocalTagCheck python:latest +# echo $? # returns '0' +# ViashDockerLocalTagCheck sdaizudceahifu +# echo $? # returns '1' +function ViashDockerLocalTagCheck { + [ -n "$(docker images -q $1)" ] +} + +# ViashDockerPull: pull a Docker image +# +# $1 : image identifier with format `[registry/]image[:tag]` +# exit code $? : whether or not the image was found +# examples: +# ViashDockerPull python:latest +# echo $? # returns '0' +# ViashDockerPull sdaizudceahifu +# echo $? # returns '1' +function ViashDockerPull { + ViashNotice "Checking if Docker image is available at '$1'" + if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then + docker pull $1 && return 0 || return 1 + else + local save=$-; set +e + docker pull $1 2> /dev/null > /dev/null + local out=$? + [[ $save =~ e ]] && set -e + if [ $out -ne 0 ]; then + ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." + fi + return $out + fi +} + +# ViashDockerPush: push a Docker image +# +# $1 : image identifier with format `[registry/]image[:tag]` +# exit code $? : whether or not the image was found +# examples: +# ViashDockerPush python:latest +# echo $? # returns '0' +# ViashDockerPush sdaizudceahifu +# echo $? # returns '1' +function ViashDockerPush { + ViashNotice "Pushing image to '$1'" + local save=$-; set +e + local out + if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then + docker push $1 + out=$? + else + docker push $1 2> /dev/null > /dev/null + out=$? + fi + [[ $save =~ e ]] && set -e + if [ $out -eq 0 ]; then + ViashNotice "Container '$1' push succeeded." + else + ViashError "Container '$1' push errored. You might not be logged in or have the necessary permissions." + fi + return $out +} + +# ViashDockerPullElseBuild: pull a Docker image, else build it +# +# $1 : image identifier with format `[registry/]image[:tag]` +# ViashDockerBuild : a Bash function which builds a docker image, takes image identifier as argument. +# examples: +# ViashDockerPullElseBuild mynewcomponent +function ViashDockerPullElseBuild { + local save=$-; set +e + ViashDockerPull $1 + local out=$? + [[ $save =~ e ]] && set -e + if [ $out -ne 0 ]; then + ViashDockerBuild $@ + fi +} + +# ViashDockerSetup: create a Docker image, according to specified docker setup strategy +# +# $1 : image identifier with format `[registry/]image[:tag]` +# $2 : docker setup strategy, see DockerSetupStrategy.scala +# examples: +# ViashDockerSetup mynewcomponent alwaysbuild +function ViashDockerSetup { + local image_id="$1" + local setup_strategy="$2" + if [ "$setup_strategy" == "alwaysbuild" -o "$setup_strategy" == "build" -o "$setup_strategy" == "b" ]; then + ViashDockerBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "alwayspull" -o "$setup_strategy" == "pull" -o "$setup_strategy" == "p" ]; then + ViashDockerPull $image_id + elif [ "$setup_strategy" == "alwayspullelsebuild" -o "$setup_strategy" == "pullelsebuild" ]; then + ViashDockerPullElseBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "alwayspullelsecachedbuild" -o "$setup_strategy" == "pullelsecachedbuild" ]; then + ViashDockerPullElseBuild $image_id $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "alwayscachedbuild" -o "$setup_strategy" == "cachedbuild" -o "$setup_strategy" == "cb" ]; then + ViashDockerBuild $image_id $(ViashDockerBuildArgs "$engine_id") + elif [[ "$setup_strategy" =~ ^ifneedbe ]]; then + local save=$-; set +e + ViashDockerLocalTagCheck $image_id + local outCheck=$? + [[ $save =~ e ]] && set -e + if [ $outCheck -eq 0 ]; then + ViashInfo "Image $image_id already exists" + elif [ "$setup_strategy" == "ifneedbebuild" ]; then + ViashDockerBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "ifneedbecachedbuild" ]; then + ViashDockerBuild $image_id $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "ifneedbepull" ]; then + ViashDockerPull $image_id + elif [ "$setup_strategy" == "ifneedbepullelsebuild" ]; then + ViashDockerPullElseBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "ifneedbepullelsecachedbuild" ]; then + ViashDockerPullElseBuild $image_id $(ViashDockerBuildArgs "$engine_id") + else + ViashError "Unrecognised Docker strategy: $setup_strategy" + exit 1 + fi + elif [ "$setup_strategy" == "push" -o "$setup_strategy" == "forcepush" -o "$setup_strategy" == "alwayspush" ]; then + ViashDockerPush "$image_id" + elif [ "$setup_strategy" == "pushifnotpresent" -o "$setup_strategy" == "gentlepush" -o "$setup_strategy" == "maybepush" ]; then + local save=$-; set +e + ViashDockerRemoteTagCheck $image_id + local outCheck=$? + [[ $save =~ e ]] && set -e + if [ $outCheck -eq 0 ]; then + ViashNotice "Container '$image_id' exists, doing nothing." + else + ViashNotice "Container '$image_id' does not yet exist." + ViashDockerPush "$image_id" + fi + elif [ "$setup_strategy" == "donothing" -o "$setup_strategy" == "meh" ]; then + ViashNotice "Skipping setup." + else + ViashError "Unrecognised Docker strategy: $setup_strategy" + exit 1 + fi +} + +# ViashDockerCheckCommands: Check whether a docker container has the required commands +# +# $1 : image identifier with format `[registry/]image[:tag]` +# $@ : commands to verify being present +# examples: +# ViashDockerCheckCommands bash:4.0 bash ps foo +function ViashDockerCheckCommands { + local image_id="$1" + shift 1 + local commands="$@" + local save=$-; set +e + local missing # mark 'missing' as local in advance, otherwise the exit code of the command will be missing and always be '0' + missing=$(docker run --rm --entrypoint=sh "$image_id" -c "for command in $commands; do command -v \$command >/dev/null 2>&1; if [ \$? -ne 0 ]; then echo \$command; exit 1; fi; done") + local outCheck=$? + [[ $save =~ e ]] && set -e + if [ $outCheck -ne 0 ]; then + ViashError "Docker container '$image_id' does not contain command '$missing'." + exit 1 + fi +} + +# ViashDockerBuild: build a docker image +# $1 : image identifier with format `[registry/]image[:tag]` +# $... : additional arguments to pass to docker build +# $VIASH_META_TEMP_DIR : temporary directory to store dockerfile & optional resources in +# $VIASH_META_NAME : name of the component +# $VIASH_META_RESOURCES_DIR : directory containing the resources +# $VIASH_VERBOSITY : verbosity level +# exit code $? : whether or not the image was built successfully +function ViashDockerBuild { + local image_id="$1" + shift 1 + + # create temporary directory to store dockerfile & optional resources in + local tmpdir=$(mktemp -d "$VIASH_META_TEMP_DIR/dockerbuild-$VIASH_META_NAME-XXXXXX") + local dockerfile="$tmpdir/Dockerfile" + function clean_up { + rm -rf "$tmpdir" + } + trap clean_up EXIT + + # store dockerfile and resources + ViashDockerfile "$VIASH_ENGINE_ID" > "$dockerfile" + + # generate the build command + local docker_build_cmd="docker build -t '$image_id' $@ '$VIASH_META_RESOURCES_DIR' -f '$dockerfile'" + + # build the container + ViashNotice "Building container '$image_id' with Dockerfile" + ViashInfo "$docker_build_cmd" + local save=$-; set +e + if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then + eval $docker_build_cmd + else + eval $docker_build_cmd &> "$tmpdir/docker_build.log" + fi + + # check exit code + local out=$? + [[ $save =~ e ]] && set -e + if [ $out -ne 0 ]; then + ViashError "Error occurred while building container '$image_id'" + if [ $VIASH_VERBOSITY -lt $VIASH_LOGCODE_INFO ]; then + ViashError "Transcript: --------------------------------" + cat "$tmpdir/docker_build.log" + ViashError "End of transcript --------------------------" + fi + exit 1 + fi +} + +######## End of helper functions for setting up Docker images for viash ######## + +# ViashDockerFile: print the dockerfile to stdout +# $1 : engine identifier +# return : dockerfile required to run this component +# examples: +# ViashDockerFile +function ViashDockerfile { + local engine_id="$1" + + if [[ "$engine_id" == "docker" ]]; then + cat << 'VIASHDOCKER' +FROM debian:stable-slim +ENTRYPOINT [] +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y bedtools procps && \ + rm -rf /var/lib/apt/lists/* + +RUN echo "bedtools: \"$(bedtools --version | sed -n 's/^bedtools //p')\"" > /var/software_versions.txt + +LABEL org.opencontainers.image.authors="Theodoro Gasperin Terra Camargo" +LABEL org.opencontainers.image.description="Companion container for running component bedtools bedtools_groupby" +LABEL org.opencontainers.image.created="2024-09-02T13:02:39Z" +LABEL org.opencontainers.image.source="https://github.com/arq5x/bedtools2" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" +LABEL org.opencontainers.image.version="main" + +VIASHDOCKER + fi +} + +# ViashDockerBuildArgs: return the arguments to pass to docker build +# $1 : engine identifier +# return : arguments to pass to docker build +function ViashDockerBuildArgs { + local engine_id="$1" + + if [[ "$engine_id" == "docker" ]]; then + echo "" + fi +} + +# ViashAbsolutePath: generate absolute path from relative path +# borrowed from https://stackoverflow.com/a/21951256 +# $1 : relative filename +# return : absolute path +# examples: +# ViashAbsolutePath some_file.txt # returns /path/to/some_file.txt +# ViashAbsolutePath /foo/bar/.. # returns /foo +function ViashAbsolutePath { + local thePath + local parr + local outp + local len + if [[ ! "$1" =~ ^/ ]]; then + thePath="$PWD/$1" + else + thePath="$1" + fi + echo "$thePath" | ( + IFS=/ + read -a parr + declare -a outp + for i in "${parr[@]}"; do + case "$i" in + ''|.) continue ;; + ..) + len=${#outp[@]} + if ((len==0)); then + continue + else + unset outp[$((len-1))] + fi + ;; + *) + len=${#outp[@]} + outp[$len]="$i" + ;; + esac + done + echo /"${outp[*]}" + ) +} +# ViashDockerAutodetectMount: auto configuring docker mounts from parameters +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts +# examples: +# ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' +# ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' +function ViashDockerAutodetectMount { + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name + if [ -d "$abs_path" ]; then + mount_source="$abs_path" + base_name="" + else + mount_source=`dirname "$abs_path"` + base_name=`basename "$abs_path"` + fi + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" + if [ -z "$base_name" ]; then + echo "$mount_target" + else + echo "$mount_target/$base_name" + fi +} +function ViashDockerAutodetectMountArg { + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name + if [ -d "$abs_path" ]; then + mount_source="$abs_path" + base_name="" + else + mount_source=`dirname "$abs_path"` + base_name=`basename "$abs_path"` + fi + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" + ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" + echo "--volume=\"$mount_source:$mount_target\"" +} +function ViashDockerStripAutomount { + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" +} +# initialise variables +VIASH_DIRECTORY_MOUNTS=() + +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + +# initialise docker variables +VIASH_DOCKER_RUN_ARGS=(-i --rm) + +# initialise array +VIASH_POSITIONAL_ARGS='' + +while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) + ViashHelp + exit + ;; + ---v|---verbose) + let "VIASH_VERBOSITY=VIASH_VERBOSITY+1" + shift 1 + ;; + ---verbosity) + VIASH_VERBOSITY="$2" + shift 2 + ;; + ---verbosity=*) + VIASH_VERBOSITY="$(ViashRemoveFlags "$1")" + shift 1 + ;; + --version) + echo "bedtools_groupby main" + exit + ;; + --input) + [ -n "$VIASH_PAR_INPUT" ] && ViashError Bad arguments for option \'--input\': \'$VIASH_PAR_INPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_INPUT="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --input. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --input=*) + [ -n "$VIASH_PAR_INPUT" ] && ViashError Bad arguments for option \'--input=*\': \'$VIASH_PAR_INPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_INPUT=$(ViashRemoveFlags "$1") + shift 1 + ;; + -i) + [ -n "$VIASH_PAR_INPUT" ] && ViashError Bad arguments for option \'-i\': \'$VIASH_PAR_INPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_INPUT="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to -i. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --output) + [ -n "$VIASH_PAR_OUTPUT" ] && ViashError Bad arguments for option \'--output\': \'$VIASH_PAR_OUTPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_OUTPUT="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --output. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --output=*) + [ -n "$VIASH_PAR_OUTPUT" ] && ViashError Bad arguments for option \'--output=*\': \'$VIASH_PAR_OUTPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_OUTPUT=$(ViashRemoveFlags "$1") + shift 1 + ;; + --groupby) + [ -n "$VIASH_PAR_GROUPBY" ] && ViashError Bad arguments for option \'--groupby\': \'$VIASH_PAR_GROUPBY\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_GROUPBY="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --groupby. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --groupby=*) + [ -n "$VIASH_PAR_GROUPBY" ] && ViashError Bad arguments for option \'--groupby=*\': \'$VIASH_PAR_GROUPBY\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_GROUPBY=$(ViashRemoveFlags "$1") + shift 1 + ;; + -g) + [ -n "$VIASH_PAR_GROUPBY" ] && ViashError Bad arguments for option \'-g\': \'$VIASH_PAR_GROUPBY\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_GROUPBY="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to -g. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + -grp) + [ -n "$VIASH_PAR_GROUPBY" ] && ViashError Bad arguments for option \'-grp\': \'$VIASH_PAR_GROUPBY\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_GROUPBY="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to -grp. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --column) + [ -n "$VIASH_PAR_COLUMN" ] && ViashError Bad arguments for option \'--column\': \'$VIASH_PAR_COLUMN\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_COLUMN="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --column. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --column=*) + [ -n "$VIASH_PAR_COLUMN" ] && ViashError Bad arguments for option \'--column=*\': \'$VIASH_PAR_COLUMN\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_COLUMN=$(ViashRemoveFlags "$1") + shift 1 + ;; + -c) + [ -n "$VIASH_PAR_COLUMN" ] && ViashError Bad arguments for option \'-c\': \'$VIASH_PAR_COLUMN\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_COLUMN="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to -c. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + -opCols) + [ -n "$VIASH_PAR_COLUMN" ] && ViashError Bad arguments for option \'-opCols\': \'$VIASH_PAR_COLUMN\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_COLUMN="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to -opCols. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --operation) + [ -n "$VIASH_PAR_OPERATION" ] && ViashError Bad arguments for option \'--operation\': \'$VIASH_PAR_OPERATION\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_OPERATION="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --operation. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --operation=*) + [ -n "$VIASH_PAR_OPERATION" ] && ViashError Bad arguments for option \'--operation=*\': \'$VIASH_PAR_OPERATION\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_OPERATION=$(ViashRemoveFlags "$1") + shift 1 + ;; + -o) + [ -n "$VIASH_PAR_OPERATION" ] && ViashError Bad arguments for option \'-o\': \'$VIASH_PAR_OPERATION\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_OPERATION="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to -o. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + -ops) + [ -n "$VIASH_PAR_OPERATION" ] && ViashError Bad arguments for option \'-ops\': \'$VIASH_PAR_OPERATION\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_OPERATION="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to -ops. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --full) + [ -n "$VIASH_PAR_FULL" ] && ViashError Bad arguments for option \'--full\': \'$VIASH_PAR_FULL\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_FULL=true + shift 1 + ;; + --inheader) + [ -n "$VIASH_PAR_INHEADER" ] && ViashError Bad arguments for option \'--inheader\': \'$VIASH_PAR_INHEADER\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_INHEADER=true + shift 1 + ;; + --outheader) + [ -n "$VIASH_PAR_OUTHEADER" ] && ViashError Bad arguments for option \'--outheader\': \'$VIASH_PAR_OUTHEADER\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_OUTHEADER=true + shift 1 + ;; + --header) + [ -n "$VIASH_PAR_HEADER" ] && ViashError Bad arguments for option \'--header\': \'$VIASH_PAR_HEADER\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_HEADER=true + shift 1 + ;; + --ignorecase) + [ -n "$VIASH_PAR_IGNORECASE" ] && ViashError Bad arguments for option \'--ignorecase\': \'$VIASH_PAR_IGNORECASE\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_IGNORECASE=true + shift 1 + ;; + --precision) + [ -n "$VIASH_PAR_PRECISION" ] && ViashError Bad arguments for option \'--precision\': \'$VIASH_PAR_PRECISION\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_PRECISION="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --precision. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --precision=*) + [ -n "$VIASH_PAR_PRECISION" ] && ViashError Bad arguments for option \'--precision=*\': \'$VIASH_PAR_PRECISION\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_PRECISION=$(ViashRemoveFlags "$1") + shift 1 + ;; + -prec) + [ -n "$VIASH_PAR_PRECISION" ] && ViashError Bad arguments for option \'-prec\': \'$VIASH_PAR_PRECISION\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_PRECISION="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to -prec. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --delimiter) + [ -n "$VIASH_PAR_DELIMITER" ] && ViashError Bad arguments for option \'--delimiter\': \'$VIASH_PAR_DELIMITER\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_DELIMITER="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --delimiter. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --delimiter=*) + [ -n "$VIASH_PAR_DELIMITER" ] && ViashError Bad arguments for option \'--delimiter=*\': \'$VIASH_PAR_DELIMITER\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_DELIMITER=$(ViashRemoveFlags "$1") + shift 1 + ;; + -delim) + [ -n "$VIASH_PAR_DELIMITER" ] && ViashError Bad arguments for option \'-delim\': \'$VIASH_PAR_DELIMITER\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_DELIMITER="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to -delim. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + ---engine) + VIASH_ENGINE_ID="$2" + shift 2 + ;; + ---engine=*) + VIASH_ENGINE_ID="$(ViashRemoveFlags "$1")" + shift 1 + ;; + ---setup) + VIASH_MODE='setup' + VIASH_SETUP_STRATEGY="$2" + shift 2 + ;; + ---setup=*) + VIASH_MODE='setup' + VIASH_SETUP_STRATEGY="$(ViashRemoveFlags "$1")" + shift 1 + ;; + ---dockerfile) + VIASH_MODE='dockerfile' + shift 1 + ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; + ---debug) + VIASH_MODE='debug' + shift 1 + ;; + ---cpus) + [ -n "$VIASH_META_CPUS" ] && ViashError Bad arguments for option \'---cpus\': \'$VIASH_META_CPUS\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_META_CPUS="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to ---cpus. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + ---cpus=*) + [ -n "$VIASH_META_CPUS" ] && ViashError Bad arguments for option \'---cpus=*\': \'$VIASH_META_CPUS\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_META_CPUS=$(ViashRemoveFlags "$1") + shift 1 + ;; + ---memory) + [ -n "$VIASH_META_MEMORY" ] && ViashError Bad arguments for option \'---memory\': \'$VIASH_META_MEMORY\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_META_MEMORY="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to ---memory. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + ---memory=*) + [ -n "$VIASH_META_MEMORY" ] && ViashError Bad arguments for option \'---memory=*\': \'$VIASH_META_MEMORY\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_META_MEMORY=$(ViashRemoveFlags "$1") + shift 1 + ;; + *) # positional arg or unknown option + # since the positional args will be eval'd, can we always quote, instead of using ViashQuote + VIASH_POSITIONAL_ARGS="$VIASH_POSITIONAL_ARGS '$1'" + [[ $1 == -* ]] && ViashWarning $1 looks like a parameter but is not a defined parameter and will instead be treated as a positional argument. Use "--help" to get more information on the parameters. + shift # past argument + ;; + esac +done + +# parse positional parameters +eval set -- $VIASH_POSITIONAL_ARGS + + +if [ "$VIASH_ENGINE_ID" == "native" ] ; then + VIASH_ENGINE_TYPE='native' +elif [ "$VIASH_ENGINE_ID" == "docker" ] ; then + VIASH_ENGINE_TYPE='docker' +else + ViashError "Engine '$VIASH_ENGINE_ID' is not recognized. Options are: docker, native." + exit 1 +fi + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + # check if docker is installed properly + ViashDockerInstallationCheck + + # determine docker image id + if [[ "$VIASH_ENGINE_ID" == 'docker' ]]; then + VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/biobox/bedtools/bedtools_groupby:main' + fi + + # print dockerfile + if [ "$VIASH_MODE" == "dockerfile" ]; then + ViashDockerfile "$VIASH_ENGINE_ID" + exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 + + # enter docker container + elif [[ "$VIASH_MODE" == "debug" ]]; then + VIASH_CMD="docker run --entrypoint=bash ${VIASH_DOCKER_RUN_ARGS[@]} -v '$(pwd)':/pwd --workdir /pwd -t $VIASH_DOCKER_IMAGE_ID" + ViashNotice "+ $VIASH_CMD" + eval $VIASH_CMD + exit + + # build docker image + elif [ "$VIASH_MODE" == "setup" ]; then + ViashDockerSetup "$VIASH_DOCKER_IMAGE_ID" "$VIASH_SETUP_STRATEGY" + ViashDockerCheckCommands "$VIASH_DOCKER_IMAGE_ID" 'ps' 'bash' + exit 0 + fi + + # check if docker image exists + ViashDockerSetup "$VIASH_DOCKER_IMAGE_ID" ifneedbepullelsecachedbuild + ViashDockerCheckCommands "$VIASH_DOCKER_IMAGE_ID" 'ps' 'bash' +fi + +# setting computational defaults + +# helper function for parsing memory strings +function ViashMemoryAsBytes { + local memory=`echo "$1" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]'` + local memory_regex='^([0-9]+)([kmgtp]i?b?|b)$' + if [[ $memory =~ $memory_regex ]]; then + local number=${memory/[^0-9]*/} + local symbol=${memory/*[0-9]/} + + case $symbol in + b) memory_b=$number ;; + kb|k) memory_b=$(( $number * 1000 )) ;; + mb|m) memory_b=$(( $number * 1000 * 1000 )) ;; + gb|g) memory_b=$(( $number * 1000 * 1000 * 1000 )) ;; + tb|t) memory_b=$(( $number * 1000 * 1000 * 1000 * 1000 )) ;; + pb|p) memory_b=$(( $number * 1000 * 1000 * 1000 * 1000 * 1000 )) ;; + kib|ki) memory_b=$(( $number * 1024 )) ;; + mib|mi) memory_b=$(( $number * 1024 * 1024 )) ;; + gib|gi) memory_b=$(( $number * 1024 * 1024 * 1024 )) ;; + tib|ti) memory_b=$(( $number * 1024 * 1024 * 1024 * 1024 )) ;; + pib|pi) memory_b=$(( $number * 1024 * 1024 * 1024 * 1024 * 1024 )) ;; + esac + echo "$memory_b" + fi +} +# compute memory in different units +if [ ! -z ${VIASH_META_MEMORY+x} ]; then + VIASH_META_MEMORY_B=`ViashMemoryAsBytes $VIASH_META_MEMORY` + # do not define other variables if memory_b is an empty string + if [ ! -z "$VIASH_META_MEMORY_B" ]; then + VIASH_META_MEMORY_KB=$(( ($VIASH_META_MEMORY_B+999) / 1000 )) + VIASH_META_MEMORY_MB=$(( ($VIASH_META_MEMORY_KB+999) / 1000 )) + VIASH_META_MEMORY_GB=$(( ($VIASH_META_MEMORY_MB+999) / 1000 )) + VIASH_META_MEMORY_TB=$(( ($VIASH_META_MEMORY_GB+999) / 1000 )) + VIASH_META_MEMORY_PB=$(( ($VIASH_META_MEMORY_TB+999) / 1000 )) + VIASH_META_MEMORY_KIB=$(( ($VIASH_META_MEMORY_B+1023) / 1024 )) + VIASH_META_MEMORY_MIB=$(( ($VIASH_META_MEMORY_KIB+1023) / 1024 )) + VIASH_META_MEMORY_GIB=$(( ($VIASH_META_MEMORY_MIB+1023) / 1024 )) + VIASH_META_MEMORY_TIB=$(( ($VIASH_META_MEMORY_GIB+1023) / 1024 )) + VIASH_META_MEMORY_PIB=$(( ($VIASH_META_MEMORY_TIB+1023) / 1024 )) + else + # unset memory if string is empty + unset $VIASH_META_MEMORY_B + fi +fi +# unset nproc if string is empty +if [ -z "$VIASH_META_CPUS" ]; then + unset $VIASH_META_CPUS +fi + + +# check whether required parameters exist +if [ -z ${VIASH_PAR_INPUT+x} ]; then + ViashError '--input' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_PAR_OUTPUT+x} ]; then + ViashError '--output' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_PAR_GROUPBY+x} ]; then + ViashError '--groupby' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_PAR_COLUMN+x} ]; then + ViashError '--column' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_NAME+x} ]; then + ViashError 'name' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_FUNCTIONALITY_NAME+x} ]; then + ViashError 'functionality_name' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_RESOURCES_DIR+x} ]; then + ViashError 'resources_dir' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_EXECUTABLE+x} ]; then + ViashError 'executable' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_CONFIG+x} ]; then + ViashError 'config' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_TEMP_DIR+x} ]; then + ViashError 'temp_dir' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi + +# filling in defaults +if [ -z ${VIASH_PAR_FULL+x} ]; then + VIASH_PAR_FULL="false" +fi +if [ -z ${VIASH_PAR_INHEADER+x} ]; then + VIASH_PAR_INHEADER="false" +fi +if [ -z ${VIASH_PAR_OUTHEADER+x} ]; then + VIASH_PAR_OUTHEADER="false" +fi +if [ -z ${VIASH_PAR_HEADER+x} ]; then + VIASH_PAR_HEADER="false" +fi +if [ -z ${VIASH_PAR_IGNORECASE+x} ]; then + VIASH_PAR_IGNORECASE="false" +fi +if [ -z ${VIASH_PAR_PRECISION+x} ]; then + VIASH_PAR_PRECISION="5" +fi +if [ -z ${VIASH_PAR_DELIMITER+x} ]; then + VIASH_PAR_DELIMITER="," +fi + +# check whether required files exist +if [ ! -z "$VIASH_PAR_INPUT" ] && [ ! -e "$VIASH_PAR_INPUT" ]; then + ViashError "Input file '$VIASH_PAR_INPUT' does not exist." + exit 1 +fi + +# check whether parameters values are of the right type +if [[ -n "$VIASH_PAR_COLUMN" ]]; then + if ! [[ "$VIASH_PAR_COLUMN" =~ ^[-+]?[0-9]+$ ]]; then + ViashError '--column' has to be an integer. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_PAR_FULL" ]]; then + if ! [[ "$VIASH_PAR_FULL" =~ ^(true|True|TRUE|false|False|FALSE|yes|Yes|YES|no|No|NO)$ ]]; then + ViashError '--full' has to be a boolean_true. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_PAR_INHEADER" ]]; then + if ! [[ "$VIASH_PAR_INHEADER" =~ ^(true|True|TRUE|false|False|FALSE|yes|Yes|YES|no|No|NO)$ ]]; then + ViashError '--inheader' has to be a boolean_true. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_PAR_OUTHEADER" ]]; then + if ! [[ "$VIASH_PAR_OUTHEADER" =~ ^(true|True|TRUE|false|False|FALSE|yes|Yes|YES|no|No|NO)$ ]]; then + ViashError '--outheader' has to be a boolean_true. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_PAR_HEADER" ]]; then + if ! [[ "$VIASH_PAR_HEADER" =~ ^(true|True|TRUE|false|False|FALSE|yes|Yes|YES|no|No|NO)$ ]]; then + ViashError '--header' has to be a boolean_true. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_PAR_IGNORECASE" ]]; then + if ! [[ "$VIASH_PAR_IGNORECASE" =~ ^(true|True|TRUE|false|False|FALSE|yes|Yes|YES|no|No|NO)$ ]]; then + ViashError '--ignorecase' has to be a boolean_true. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_PAR_PRECISION" ]]; then + if ! [[ "$VIASH_PAR_PRECISION" =~ ^[-+]?[0-9]+$ ]]; then + ViashError '--precision' has to be an integer. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_CPUS" ]]; then + if ! [[ "$VIASH_META_CPUS" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'cpus' has to be an integer. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_B" ]]; then + if ! [[ "$VIASH_META_MEMORY_B" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_b' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_KB" ]]; then + if ! [[ "$VIASH_META_MEMORY_KB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_kb' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_MB" ]]; then + if ! [[ "$VIASH_META_MEMORY_MB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_mb' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_GB" ]]; then + if ! [[ "$VIASH_META_MEMORY_GB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_gb' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_TB" ]]; then + if ! [[ "$VIASH_META_MEMORY_TB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_tb' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_PB" ]]; then + if ! [[ "$VIASH_META_MEMORY_PB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_pb' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_KIB" ]]; then + if ! [[ "$VIASH_META_MEMORY_KIB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_kib' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_MIB" ]]; then + if ! [[ "$VIASH_META_MEMORY_MIB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_mib' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_GIB" ]]; then + if ! [[ "$VIASH_META_MEMORY_GIB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_gib' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_TIB" ]]; then + if ! [[ "$VIASH_META_MEMORY_TIB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_tib' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_PIB" ]]; then + if ! [[ "$VIASH_META_MEMORY_PIB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_pib' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi + +# create parent directories of output files, if so desired +if [ ! -z "$VIASH_PAR_OUTPUT" ] && [ ! -d "$(dirname "$VIASH_PAR_OUTPUT")" ]; then + mkdir -p "$(dirname "$VIASH_PAR_OUTPUT")" +fi + +if [ "$VIASH_ENGINE_ID" == "native" ] ; then + if [ "$VIASH_MODE" == "run" ]; then + VIASH_CMD="bash" + else + ViashError "Engine '$VIASH_ENGINE_ID' does not support mode '$VIASH_MODE'." + exit 1 + fi +fi + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + # detect volumes from file arguments + VIASH_CHOWN_VARS=() +if [ ! -z "$VIASH_PAR_INPUT" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_PAR_INPUT")" ) + VIASH_PAR_INPUT=$(ViashDockerAutodetectMount "$VIASH_PAR_INPUT") +fi +if [ ! -z "$VIASH_PAR_OUTPUT" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_PAR_OUTPUT")" ) + VIASH_PAR_OUTPUT=$(ViashDockerAutodetectMount "$VIASH_PAR_OUTPUT") + VIASH_CHOWN_VARS+=( "$VIASH_PAR_OUTPUT" ) +fi +if [ ! -z "$VIASH_META_RESOURCES_DIR" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_RESOURCES_DIR")" ) + VIASH_META_RESOURCES_DIR=$(ViashDockerAutodetectMount "$VIASH_META_RESOURCES_DIR") +fi +if [ ! -z "$VIASH_META_EXECUTABLE" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_EXECUTABLE")" ) + VIASH_META_EXECUTABLE=$(ViashDockerAutodetectMount "$VIASH_META_EXECUTABLE") +fi +if [ ! -z "$VIASH_META_CONFIG" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_CONFIG")" ) + VIASH_META_CONFIG=$(ViashDockerAutodetectMount "$VIASH_META_CONFIG") +fi +if [ ! -z "$VIASH_META_TEMP_DIR" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_TEMP_DIR")" ) + VIASH_META_TEMP_DIR=$(ViashDockerAutodetectMount "$VIASH_META_TEMP_DIR") +fi + + # get unique mounts + VIASH_UNIQUE_MOUNTS=($(for val in "${VIASH_DIRECTORY_MOUNTS[@]}"; do echo "$val"; done | sort -u)) +fi + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + # change file ownership + function ViashPerformChown { + if (( ${#VIASH_CHOWN_VARS[@]} )); then + set +e + VIASH_CMD="docker run --entrypoint=bash --rm ${VIASH_UNIQUE_MOUNTS[@]} $VIASH_DOCKER_IMAGE_ID -c 'chown $(id -u):$(id -g) --silent --recursive ${VIASH_CHOWN_VARS[@]}'" + ViashDebug "+ $VIASH_CMD" + eval $VIASH_CMD + set -e + fi + } + trap ViashPerformChown EXIT +fi + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + # helper function for filling in extra docker args + if [ ! -z "$VIASH_META_MEMORY_B" ]; then + VIASH_DOCKER_RUN_ARGS+=("--memory=${VIASH_META_MEMORY_B}") + fi + if [ ! -z "$VIASH_META_CPUS" ]; then + VIASH_DOCKER_RUN_ARGS+=("--cpus=${VIASH_META_CPUS}") + fi +fi + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + VIASH_CMD="docker run --entrypoint=bash ${VIASH_DOCKER_RUN_ARGS[@]} ${VIASH_UNIQUE_MOUNTS[@]} $VIASH_DOCKER_IMAGE_ID" +fi + + +# set dependency paths + + +ViashDebug "Running command: $(echo $VIASH_CMD)" +cat << VIASHEOF | eval $VIASH_CMD +set -e +tempscript=\$(mktemp "$VIASH_META_TEMP_DIR/viash-run-bedtools_groupby-XXXXXX").sh +function clean_up { + rm "\$tempscript" +} +function interrupt { + echo -e "\nCTRL-C Pressed..." + exit 1 +} +trap clean_up EXIT +trap interrupt INT SIGINT +cat > "\$tempscript" << 'VIASHMAIN' +#!/bin/bash + +## VIASH START +# The following code has been auto-generated by Viash. +$( if [ ! -z ${VIASH_PAR_INPUT+x} ]; then echo "${VIASH_PAR_INPUT}" | sed "s#'#'\"'\"'#g;s#.*#par_input='&'#" ; else echo "# par_input="; fi ) +$( if [ ! -z ${VIASH_PAR_OUTPUT+x} ]; then echo "${VIASH_PAR_OUTPUT}" | sed "s#'#'\"'\"'#g;s#.*#par_output='&'#" ; else echo "# par_output="; fi ) +$( if [ ! -z ${VIASH_PAR_GROUPBY+x} ]; then echo "${VIASH_PAR_GROUPBY}" | sed "s#'#'\"'\"'#g;s#.*#par_groupby='&'#" ; else echo "# par_groupby="; fi ) +$( if [ ! -z ${VIASH_PAR_COLUMN+x} ]; then echo "${VIASH_PAR_COLUMN}" | sed "s#'#'\"'\"'#g;s#.*#par_column='&'#" ; else echo "# par_column="; fi ) +$( if [ ! -z ${VIASH_PAR_OPERATION+x} ]; then echo "${VIASH_PAR_OPERATION}" | sed "s#'#'\"'\"'#g;s#.*#par_operation='&'#" ; else echo "# par_operation="; fi ) +$( if [ ! -z ${VIASH_PAR_FULL+x} ]; then echo "${VIASH_PAR_FULL}" | sed "s#'#'\"'\"'#g;s#.*#par_full='&'#" ; else echo "# par_full="; fi ) +$( if [ ! -z ${VIASH_PAR_INHEADER+x} ]; then echo "${VIASH_PAR_INHEADER}" | sed "s#'#'\"'\"'#g;s#.*#par_inheader='&'#" ; else echo "# par_inheader="; fi ) +$( if [ ! -z ${VIASH_PAR_OUTHEADER+x} ]; then echo "${VIASH_PAR_OUTHEADER}" | sed "s#'#'\"'\"'#g;s#.*#par_outheader='&'#" ; else echo "# par_outheader="; fi ) +$( if [ ! -z ${VIASH_PAR_HEADER+x} ]; then echo "${VIASH_PAR_HEADER}" | sed "s#'#'\"'\"'#g;s#.*#par_header='&'#" ; else echo "# par_header="; fi ) +$( if [ ! -z ${VIASH_PAR_IGNORECASE+x} ]; then echo "${VIASH_PAR_IGNORECASE}" | sed "s#'#'\"'\"'#g;s#.*#par_ignorecase='&'#" ; else echo "# par_ignorecase="; fi ) +$( if [ ! -z ${VIASH_PAR_PRECISION+x} ]; then echo "${VIASH_PAR_PRECISION}" | sed "s#'#'\"'\"'#g;s#.*#par_precision='&'#" ; else echo "# par_precision="; fi ) +$( if [ ! -z ${VIASH_PAR_DELIMITER+x} ]; then echo "${VIASH_PAR_DELIMITER}" | sed "s#'#'\"'\"'#g;s#.*#par_delimiter='&'#" ; else echo "# par_delimiter="; fi ) +$( if [ ! -z ${VIASH_META_NAME+x} ]; then echo "${VIASH_META_NAME}" | sed "s#'#'\"'\"'#g;s#.*#meta_name='&'#" ; else echo "# meta_name="; fi ) +$( if [ ! -z ${VIASH_META_FUNCTIONALITY_NAME+x} ]; then echo "${VIASH_META_FUNCTIONALITY_NAME}" | sed "s#'#'\"'\"'#g;s#.*#meta_functionality_name='&'#" ; else echo "# meta_functionality_name="; fi ) +$( if [ ! -z ${VIASH_META_RESOURCES_DIR+x} ]; then echo "${VIASH_META_RESOURCES_DIR}" | sed "s#'#'\"'\"'#g;s#.*#meta_resources_dir='&'#" ; else echo "# meta_resources_dir="; fi ) +$( if [ ! -z ${VIASH_META_EXECUTABLE+x} ]; then echo "${VIASH_META_EXECUTABLE}" | sed "s#'#'\"'\"'#g;s#.*#meta_executable='&'#" ; else echo "# meta_executable="; fi ) +$( if [ ! -z ${VIASH_META_CONFIG+x} ]; then echo "${VIASH_META_CONFIG}" | sed "s#'#'\"'\"'#g;s#.*#meta_config='&'#" ; else echo "# meta_config="; fi ) +$( if [ ! -z ${VIASH_META_TEMP_DIR+x} ]; then echo "${VIASH_META_TEMP_DIR}" | sed "s#'#'\"'\"'#g;s#.*#meta_temp_dir='&'#" ; else echo "# meta_temp_dir="; fi ) +$( if [ ! -z ${VIASH_META_CPUS+x} ]; then echo "${VIASH_META_CPUS}" | sed "s#'#'\"'\"'#g;s#.*#meta_cpus='&'#" ; else echo "# meta_cpus="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_B+x} ]; then echo "${VIASH_META_MEMORY_B}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_b='&'#" ; else echo "# meta_memory_b="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_KB+x} ]; then echo "${VIASH_META_MEMORY_KB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_kb='&'#" ; else echo "# meta_memory_kb="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_MB+x} ]; then echo "${VIASH_META_MEMORY_MB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_mb='&'#" ; else echo "# meta_memory_mb="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_GB+x} ]; then echo "${VIASH_META_MEMORY_GB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_gb='&'#" ; else echo "# meta_memory_gb="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_TB+x} ]; then echo "${VIASH_META_MEMORY_TB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_tb='&'#" ; else echo "# meta_memory_tb="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_PB+x} ]; then echo "${VIASH_META_MEMORY_PB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_pb='&'#" ; else echo "# meta_memory_pb="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_KIB+x} ]; then echo "${VIASH_META_MEMORY_KIB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_kib='&'#" ; else echo "# meta_memory_kib="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_MIB+x} ]; then echo "${VIASH_META_MEMORY_MIB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_mib='&'#" ; else echo "# meta_memory_mib="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_GIB+x} ]; then echo "${VIASH_META_MEMORY_GIB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_gib='&'#" ; else echo "# meta_memory_gib="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_TIB+x} ]; then echo "${VIASH_META_MEMORY_TIB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_tib='&'#" ; else echo "# meta_memory_tib="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_PIB+x} ]; then echo "${VIASH_META_MEMORY_PIB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_pib='&'#" ; else echo "# meta_memory_pib="; fi ) + +## VIASH END + +# Exit on error +set -eo pipefail + +# Unset parameters +unset_if_false=( + par_full + par_inheader + par_outheader + par_header + par_ignorecase +) + +for par in \${unset_if_false[@]}; do + test_val="\${!par}" + [[ "\$test_val" == "false" ]] && unset \$par +done + +bedtools groupby \\ + \${par_full:+-full} \\ + \${par_inheader:+-inheader} \\ + \${par_outheader:+-outheader} \\ + \${par_header:+-header} \\ + \${par_ignorecase:+-ignorecase} \\ + \${par_precision:+-prec "\$par_precision"} \\ + \${par_delimiter:+-delim "\$par_delimiter"} \\ + -i "\$par_input" \\ + -g "\$par_groupby" \\ + -c "\$par_column" \\ + \${par_operation:+-o "\$par_operation"} \\ + > "\$par_output" + +VIASHMAIN +bash "\$tempscript" & +wait "\$!" + +VIASHEOF + + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + # strip viash automount from file paths + + if [ ! -z "$VIASH_PAR_INPUT" ]; then + VIASH_PAR_INPUT=$(ViashDockerStripAutomount "$VIASH_PAR_INPUT") + fi + if [ ! -z "$VIASH_PAR_OUTPUT" ]; then + VIASH_PAR_OUTPUT=$(ViashDockerStripAutomount "$VIASH_PAR_OUTPUT") + fi + if [ ! -z "$VIASH_META_RESOURCES_DIR" ]; then + VIASH_META_RESOURCES_DIR=$(ViashDockerStripAutomount "$VIASH_META_RESOURCES_DIR") + fi + if [ ! -z "$VIASH_META_EXECUTABLE" ]; then + VIASH_META_EXECUTABLE=$(ViashDockerStripAutomount "$VIASH_META_EXECUTABLE") + fi + if [ ! -z "$VIASH_META_CONFIG" ]; then + VIASH_META_CONFIG=$(ViashDockerStripAutomount "$VIASH_META_CONFIG") + fi + if [ ! -z "$VIASH_META_TEMP_DIR" ]; then + VIASH_META_TEMP_DIR=$(ViashDockerStripAutomount "$VIASH_META_TEMP_DIR") + fi +fi + + +# check whether required files exist +if [ ! -z "$VIASH_PAR_OUTPUT" ] && [ ! -e "$VIASH_PAR_OUTPUT" ]; then + ViashError "Output file '$VIASH_PAR_OUTPUT' does not exist." + exit 1 +fi + + +exit 0 diff --git a/target/executable/bedtools/bedtools_intersect/.config.vsh.yaml b/target/executable/bedtools/bedtools_intersect/.config.vsh.yaml index 71cbbe55..01c73f6f 100644 --- a/target/executable/bedtools/bedtools_intersect/.config.vsh.yaml +++ b/target/executable/bedtools/bedtools_intersect/.config.vsh.yaml @@ -409,15 +409,15 @@ build_info: engine: "docker|native" output: "target/executable/bedtools/bedtools_intersect" executable: "target/executable/bedtools/bedtools_intersect/bedtools_intersect" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/bedtools/bedtools_intersect/bedtools_intersect b/target/executable/bedtools/bedtools_intersect/bedtools_intersect index 61d28eed..1b41d7d2 100755 --- a/target/executable/bedtools/bedtools_intersect/bedtools_intersect +++ b/target/executable/bedtools/bedtools_intersect/bedtools_intersect @@ -2,7 +2,7 @@ # bedtools_intersect main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -57,24 +57,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -369,9 +369,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -423,9 +423,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -445,7 +445,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -469,9 +470,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -632,9 +633,9 @@ RUN echo "bedtools: \"$(bedtools --version | sed -n 's/^bedtools //p')\"" > /var LABEL org.opencontainers.image.authors="Theodoro Gasperin Terra Camargo" LABEL org.opencontainers.image.description="Companion container for running component bedtools bedtools_intersect" -LABEL org.opencontainers.image.created="2024-08-21T11:35:51Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:38Z" LABEL org.opencontainers.image.source="https://github.com/arq5x/bedtools2" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -661,6 +662,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -691,14 +695,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -706,7 +713,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -714,7 +721,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -722,17 +731,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -1091,6 +1105,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -1152,6 +1178,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/bedtools/bedtools_links/.config.vsh.yaml b/target/executable/bedtools/bedtools_links/.config.vsh.yaml new file mode 100644 index 00000000..97f9d3f6 --- /dev/null +++ b/target/executable/bedtools/bedtools_links/.config.vsh.yaml @@ -0,0 +1,236 @@ +name: "bedtools_links" +namespace: "bedtools" +version: "main" +authors: +- name: "Theodoro Gasperin Terra Camargo" + roles: + - "author" + - "maintainer" + info: + links: + email: "theodorogtc@gmail.com" + github: "tgaspe" + linkedin: "theodoro-gasperin-terra-camargo" + organizations: + - name: "Data Intuitive" + href: "https://www.data-intuitive.com" + role: "Bioinformatician" +argument_groups: +- name: "Inputs" + arguments: + - type: "file" + name: "--input" + alternatives: + - "-i" + description: "Input file (bed/gff/vcf)." + info: null + must_exist: true + create_parent: true + required: true + direction: "input" + multiple: false + multiple_sep: ";" +- name: "Outputs" + arguments: + - type: "file" + name: "--output" + alternatives: + - "-o" + description: "Output HTML file to be written." + info: null + must_exist: true + create_parent: true + required: false + direction: "output" + multiple: false + multiple_sep: ";" +- name: "Options" + description: "By default, the links created will point to human (hg18) UCSC browser.\n\ + If you have a local mirror, you can override this behavior by supplying\nthe -base,\ + \ -org, and -db options.\n\nFor example, if the URL of your local mirror for mouse\ + \ MM9 is called: \nhttp://mymirror.myuniversity.edu, then you would use the following:\n\ + --base_url http://mymirror.myuniversity.edu\n--organism mouse\n--database mm9\n" + arguments: + - type: "string" + name: "--base_url" + alternatives: + - "-base" + description: "The “basename” for the UCSC browser.\n" + info: null + default: + - "http://genome.ucsc.edu" + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "string" + name: "--organism" + alternatives: + - "-org" + description: "The organism (e.g. mouse, human). \n" + info: null + default: + - "human" + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "string" + name: "--database" + alternatives: + - "-db" + description: "The genome build. \n" + info: null + default: + - "hg18" + required: false + direction: "input" + multiple: false + multiple_sep: ";" +resources: +- type: "bash_script" + path: "script.sh" + is_executable: true +description: "Creates an HTML file with links to an instance of the UCSC Genome Browser\ + \ for all features / intervals in a file. \nThis is useful for cases when one wants\ + \ to manually inspect through a large set of annotations or features.\n" +test_resources: +- type: "bash_script" + path: "test.sh" + is_executable: true +info: null +status: "enabled" +requirements: + commands: + - "ps" +keywords: +- "Links" +- "BED" +- "GFF" +- "VCF" +license: "MIT" +references: + doi: + - "10.1093/bioinformatics/btq033" +links: + repository: "https://github.com/arq5x/bedtools2" + homepage: "https://bedtools.readthedocs.io/en/latest/#" + documentation: "https://bedtools.readthedocs.io/en/latest/content/tools/links.html" + issue_tracker: "https://github.com/arq5x/bedtools2/issues" +runners: +- type: "executable" + id: "executable" + docker_setup_strategy: "ifneedbepullelsecachedbuild" +- type: "nextflow" + id: "nextflow" + directives: + tag: "$id" + auto: + simplifyInput: true + simplifyOutput: false + transcript: false + publish: false + config: + labels: + mem1gb: "memory = 1000000000.B" + mem2gb: "memory = 2000000000.B" + mem5gb: "memory = 5000000000.B" + mem10gb: "memory = 10000000000.B" + mem20gb: "memory = 20000000000.B" + mem50gb: "memory = 50000000000.B" + mem100gb: "memory = 100000000000.B" + mem200gb: "memory = 200000000000.B" + mem500gb: "memory = 500000000000.B" + mem1tb: "memory = 1000000000000.B" + mem2tb: "memory = 2000000000000.B" + mem5tb: "memory = 5000000000000.B" + mem10tb: "memory = 10000000000000.B" + mem20tb: "memory = 20000000000000.B" + mem50tb: "memory = 50000000000000.B" + mem100tb: "memory = 100000000000000.B" + mem200tb: "memory = 200000000000000.B" + mem500tb: "memory = 500000000000000.B" + mem1gib: "memory = 1073741824.B" + mem2gib: "memory = 2147483648.B" + mem4gib: "memory = 4294967296.B" + mem8gib: "memory = 8589934592.B" + mem16gib: "memory = 17179869184.B" + mem32gib: "memory = 34359738368.B" + mem64gib: "memory = 68719476736.B" + mem128gib: "memory = 137438953472.B" + mem256gib: "memory = 274877906944.B" + mem512gib: "memory = 549755813888.B" + mem1tib: "memory = 1099511627776.B" + mem2tib: "memory = 2199023255552.B" + mem4tib: "memory = 4398046511104.B" + mem8tib: "memory = 8796093022208.B" + mem16tib: "memory = 17592186044416.B" + mem32tib: "memory = 35184372088832.B" + mem64tib: "memory = 70368744177664.B" + mem128tib: "memory = 140737488355328.B" + mem256tib: "memory = 281474976710656.B" + mem512tib: "memory = 562949953421312.B" + cpu1: "cpus = 1" + cpu2: "cpus = 2" + cpu5: "cpus = 5" + cpu10: "cpus = 10" + cpu20: "cpus = 20" + cpu50: "cpus = 50" + cpu100: "cpus = 100" + cpu200: "cpus = 200" + cpu500: "cpus = 500" + cpu1000: "cpus = 1000" + debug: false + container: "docker" +engines: +- type: "docker" + id: "docker" + image: "debian:stable-slim" + target_registry: "images.viash-hub.com" + target_tag: "main" + namespace_separator: "/" + setup: + - type: "apt" + packages: + - "bedtools" + - "procps" + interactive: false + - type: "docker" + run: + - "echo \"bedtools: \\\"$(bedtools --version | sed -n 's/^bedtools //p')\\\"\"\ + \ > /var/software_versions.txt\n" + entrypoint: [] + cmd: null +- type: "native" + id: "native" +build_info: + config: "src/bedtools/bedtools_links/config.vsh.yaml" + runner: "executable" + engine: "docker|native" + output: "target/executable/bedtools/bedtools_links" + executable: "target/executable/bedtools/bedtools_links/bedtools_links" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" + git_remote: "https://github.com/viash-hub/biobox" +package_config: + name: "biobox" + version: "main" + description: "A collection of bioinformatics tools for working with sequence data.\n" + info: null + viash_version: "0.9.0-RC7" + source: "src" + target: "target" + config_mods: + - ".requirements.commands := ['ps']\n" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'main'" + keywords: + - "bioinformatics" + - "modules" + - "sequencing" + license: "MIT" + organization: "vsh" + links: + repository: "https://github.com/viash-hub/biobox" + issue_tracker: "https://github.com/viash-hub/biobox/issues" diff --git a/target/executable/bedtools/bedtools_links/bedtools_links b/target/executable/bedtools/bedtools_links/bedtools_links new file mode 100755 index 00000000..6a12b689 --- /dev/null +++ b/target/executable/bedtools/bedtools_links/bedtools_links @@ -0,0 +1,1192 @@ +#!/usr/bin/env bash + +# bedtools_links main +# +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a +# derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from +# Data Intuitive. +# +# The component may contain files which fall under a different license. The +# authors of this component should specify the license in the header of such +# files, or include a separate license file detailing the licenses of all included +# files. +# +# Component authors: +# * Theodoro Gasperin Terra Camargo (author, maintainer) + +set -e + +if [ -z "$VIASH_TEMP" ]; then + VIASH_TEMP=${VIASH_TEMP:-$VIASH_TMPDIR} + VIASH_TEMP=${VIASH_TEMP:-$VIASH_TEMPDIR} + VIASH_TEMP=${VIASH_TEMP:-$VIASH_TMP} + VIASH_TEMP=${VIASH_TEMP:-$TMPDIR} + VIASH_TEMP=${VIASH_TEMP:-$TMP} + VIASH_TEMP=${VIASH_TEMP:-$TEMPDIR} + VIASH_TEMP=${VIASH_TEMP:-$TEMP} + VIASH_TEMP=${VIASH_TEMP:-/tmp} +fi + +# define helper functions +# ViashQuote: put quotes around non flag values +# $1 : unquoted string +# return : possibly quoted string +# examples: +# ViashQuote --foo # returns --foo +# ViashQuote bar # returns 'bar' +# Viashquote --foo=bar # returns --foo='bar' +function ViashQuote { + if [[ "$1" =~ ^-+[a-zA-Z0-9_\-]+=.+$ ]]; then + echo "$1" | sed "s#=\(.*\)#='\1'#" + elif [[ "$1" =~ ^-+[a-zA-Z0-9_\-]+$ ]]; then + echo "$1" + else + echo "'$1'" + fi +} +# ViashRemoveFlags: Remove leading flag +# $1 : string with a possible leading flag +# return : string without possible leading flag +# examples: +# ViashRemoveFlags --foo=bar # returns bar +function ViashRemoveFlags { + echo "$1" | sed 's/^--*[a-zA-Z0-9_\-]*=//' +} +# ViashSourceDir: return the path of a bash file, following symlinks +# usage : ViashSourceDir ${BASH_SOURCE[0]} +# $1 : Should always be set to ${BASH_SOURCE[0]} +# returns : The absolute path of the bash file +function ViashSourceDir { + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" + done + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd +} +# ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks +# usage : ViashFindTargetDir 'ScriptPath' +# $1 : The location from where to start the upward search +# returns : The absolute path of the '.build.yaml' file +function ViashFindTargetDir { + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} + done + echo $source +} +# see https://en.wikipedia.org/wiki/Syslog#Severity_level +VIASH_LOGCODE_EMERGENCY=0 +VIASH_LOGCODE_ALERT=1 +VIASH_LOGCODE_CRITICAL=2 +VIASH_LOGCODE_ERROR=3 +VIASH_LOGCODE_WARNING=4 +VIASH_LOGCODE_NOTICE=5 +VIASH_LOGCODE_INFO=6 +VIASH_LOGCODE_DEBUG=7 +VIASH_VERBOSITY=$VIASH_LOGCODE_NOTICE + +# ViashLog: Log events depending on the verbosity level +# usage: ViashLog 1 alert Oh no something went wrong! +# $1: required verbosity level +# $2: display tag +# $3+: messages to display +# stdout: Your input, prepended by '[$2] '. +function ViashLog { + local required_level="$1" + local display_tag="$2" + shift 2 + if [ $VIASH_VERBOSITY -ge $required_level ]; then + >&2 echo "[$display_tag]" "$@" + fi +} + +# ViashEmergency: log events when the system is unstable +# usage: ViashEmergency Oh no something went wrong. +# stdout: Your input, prepended by '[emergency] '. +function ViashEmergency { + ViashLog $VIASH_LOGCODE_EMERGENCY emergency "$@" +} + +# ViashAlert: log events when actions must be taken immediately (e.g. corrupted system database) +# usage: ViashAlert Oh no something went wrong. +# stdout: Your input, prepended by '[alert] '. +function ViashAlert { + ViashLog $VIASH_LOGCODE_ALERT alert "$@" +} + +# ViashCritical: log events when a critical condition occurs +# usage: ViashCritical Oh no something went wrong. +# stdout: Your input, prepended by '[critical] '. +function ViashCritical { + ViashLog $VIASH_LOGCODE_CRITICAL critical "$@" +} + +# ViashError: log events when an error condition occurs +# usage: ViashError Oh no something went wrong. +# stdout: Your input, prepended by '[error] '. +function ViashError { + ViashLog $VIASH_LOGCODE_ERROR error "$@" +} + +# ViashWarning: log potentially abnormal events +# usage: ViashWarning Something may have gone wrong. +# stdout: Your input, prepended by '[warning] '. +function ViashWarning { + ViashLog $VIASH_LOGCODE_WARNING warning "$@" +} + +# ViashNotice: log significant but normal events +# usage: ViashNotice This just happened. +# stdout: Your input, prepended by '[notice] '. +function ViashNotice { + ViashLog $VIASH_LOGCODE_NOTICE notice "$@" +} + +# ViashInfo: log normal events +# usage: ViashInfo This just happened. +# stdout: Your input, prepended by '[info] '. +function ViashInfo { + ViashLog $VIASH_LOGCODE_INFO info "$@" +} + +# ViashDebug: log all events, for debugging purposes +# usage: ViashDebug This just happened. +# stdout: Your input, prepended by '[debug] '. +function ViashDebug { + ViashLog $VIASH_LOGCODE_DEBUG debug "$@" +} + +# find source folder of this component +VIASH_META_RESOURCES_DIR=`ViashSourceDir ${BASH_SOURCE[0]}` + +# find the root of the built components & dependencies +VIASH_TARGET_DIR=`ViashFindTargetDir $VIASH_META_RESOURCES_DIR` + +# define meta fields +VIASH_META_NAME="bedtools_links" +VIASH_META_FUNCTIONALITY_NAME="bedtools_links" +VIASH_META_EXECUTABLE="$VIASH_META_RESOURCES_DIR/$VIASH_META_NAME" +VIASH_META_CONFIG="$VIASH_META_RESOURCES_DIR/.config.vsh.yaml" +VIASH_META_TEMP_DIR="$VIASH_TEMP" + + +# ViashHelp: Display helpful explanation about this executable +function ViashHelp { + echo "bedtools_links main" + echo "" + echo "Creates an HTML file with links to an instance of the UCSC Genome Browser for" + echo "all features / intervals in a file." + echo "This is useful for cases when one wants to manually inspect through a large set" + echo "of annotations or features." + echo "" + echo "Inputs:" + echo " -i, --input" + echo " type: file, required parameter, file must exist" + echo " Input file (bed/gff/vcf)." + echo "" + echo "Outputs:" + echo " -o, --output" + echo " type: file, output, file must exist" + echo " Output HTML file to be written." + echo "" + echo "Options:" + echo " By default, the links created will point to human (hg18) UCSC browser." + echo " If you have a local mirror, you can override this behavior by supplying" + echo " the -base, -org, and -db options." + echo " For example, if the URL of your local mirror for mouse MM9 is called:" + echo " http://mymirror.myuniversity.edu, then you would use the following:" + echo " --base_url http://mymirror.myuniversity.edu" + echo " --organism mouse" + echo " --database mm9" + echo "" + echo " -base, --base_url" + echo " type: string" + echo " default: http://genome.ucsc.edu" + echo " The “basename” for the UCSC browser." + echo "" + echo " -org, --organism" + echo " type: string" + echo " default: human" + echo " The organism (e.g. mouse, human)." + echo "" + echo " -db, --database" + echo " type: string" + echo " default: hg18" + echo " The genome build." +} + +# initialise variables +VIASH_MODE='run' +VIASH_ENGINE_ID='docker' + +######## Helper functions for setting up Docker images for viash ######## +# expects: ViashDockerBuild + +# ViashDockerInstallationCheck: check whether Docker is installed correctly +# +# examples: +# ViashDockerInstallationCheck +function ViashDockerInstallationCheck { + ViashDebug "Checking whether Docker is installed" + if [ ! command -v docker &> /dev/null ]; then + ViashCritical "Docker doesn't seem to be installed. See 'https://docs.docker.com/get-docker/' for instructions." + exit 1 + fi + + ViashDebug "Checking whether the Docker daemon is running" + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? + [[ $save =~ e ]] && set -e + if [ $out -ne 0 ]; then + ViashCritical "Docker daemon does not seem to be running. Try one of the following:" + ViashCritical "- Try running 'dockerd' in the command line" + ViashCritical "- See https://docs.docker.com/config/daemon/" + exit 1 + fi +} + +# ViashDockerRemoteTagCheck: check whether a Docker image is available +# on a remote. Assumes `docker login` has been performed, if relevant. +# +# $1 : image identifier with format `[registry/]image[:tag]` +# exit code $? : whether or not the image was found +# examples: +# ViashDockerRemoteTagCheck python:latest +# echo $? # returns '0' +# ViashDockerRemoteTagCheck sdaizudceahifu +# echo $? # returns '1' +function ViashDockerRemoteTagCheck { + docker manifest inspect $1 > /dev/null 2> /dev/null +} + +# ViashDockerLocalTagCheck: check whether a Docker image is available locally +# +# $1 : image identifier with format `[registry/]image[:tag]` +# exit code $? : whether or not the image was found +# examples: +# docker pull python:latest +# ViashDockerLocalTagCheck python:latest +# echo $? # returns '0' +# ViashDockerLocalTagCheck sdaizudceahifu +# echo $? # returns '1' +function ViashDockerLocalTagCheck { + [ -n "$(docker images -q $1)" ] +} + +# ViashDockerPull: pull a Docker image +# +# $1 : image identifier with format `[registry/]image[:tag]` +# exit code $? : whether or not the image was found +# examples: +# ViashDockerPull python:latest +# echo $? # returns '0' +# ViashDockerPull sdaizudceahifu +# echo $? # returns '1' +function ViashDockerPull { + ViashNotice "Checking if Docker image is available at '$1'" + if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then + docker pull $1 && return 0 || return 1 + else + local save=$-; set +e + docker pull $1 2> /dev/null > /dev/null + local out=$? + [[ $save =~ e ]] && set -e + if [ $out -ne 0 ]; then + ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." + fi + return $out + fi +} + +# ViashDockerPush: push a Docker image +# +# $1 : image identifier with format `[registry/]image[:tag]` +# exit code $? : whether or not the image was found +# examples: +# ViashDockerPush python:latest +# echo $? # returns '0' +# ViashDockerPush sdaizudceahifu +# echo $? # returns '1' +function ViashDockerPush { + ViashNotice "Pushing image to '$1'" + local save=$-; set +e + local out + if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then + docker push $1 + out=$? + else + docker push $1 2> /dev/null > /dev/null + out=$? + fi + [[ $save =~ e ]] && set -e + if [ $out -eq 0 ]; then + ViashNotice "Container '$1' push succeeded." + else + ViashError "Container '$1' push errored. You might not be logged in or have the necessary permissions." + fi + return $out +} + +# ViashDockerPullElseBuild: pull a Docker image, else build it +# +# $1 : image identifier with format `[registry/]image[:tag]` +# ViashDockerBuild : a Bash function which builds a docker image, takes image identifier as argument. +# examples: +# ViashDockerPullElseBuild mynewcomponent +function ViashDockerPullElseBuild { + local save=$-; set +e + ViashDockerPull $1 + local out=$? + [[ $save =~ e ]] && set -e + if [ $out -ne 0 ]; then + ViashDockerBuild $@ + fi +} + +# ViashDockerSetup: create a Docker image, according to specified docker setup strategy +# +# $1 : image identifier with format `[registry/]image[:tag]` +# $2 : docker setup strategy, see DockerSetupStrategy.scala +# examples: +# ViashDockerSetup mynewcomponent alwaysbuild +function ViashDockerSetup { + local image_id="$1" + local setup_strategy="$2" + if [ "$setup_strategy" == "alwaysbuild" -o "$setup_strategy" == "build" -o "$setup_strategy" == "b" ]; then + ViashDockerBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "alwayspull" -o "$setup_strategy" == "pull" -o "$setup_strategy" == "p" ]; then + ViashDockerPull $image_id + elif [ "$setup_strategy" == "alwayspullelsebuild" -o "$setup_strategy" == "pullelsebuild" ]; then + ViashDockerPullElseBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "alwayspullelsecachedbuild" -o "$setup_strategy" == "pullelsecachedbuild" ]; then + ViashDockerPullElseBuild $image_id $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "alwayscachedbuild" -o "$setup_strategy" == "cachedbuild" -o "$setup_strategy" == "cb" ]; then + ViashDockerBuild $image_id $(ViashDockerBuildArgs "$engine_id") + elif [[ "$setup_strategy" =~ ^ifneedbe ]]; then + local save=$-; set +e + ViashDockerLocalTagCheck $image_id + local outCheck=$? + [[ $save =~ e ]] && set -e + if [ $outCheck -eq 0 ]; then + ViashInfo "Image $image_id already exists" + elif [ "$setup_strategy" == "ifneedbebuild" ]; then + ViashDockerBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "ifneedbecachedbuild" ]; then + ViashDockerBuild $image_id $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "ifneedbepull" ]; then + ViashDockerPull $image_id + elif [ "$setup_strategy" == "ifneedbepullelsebuild" ]; then + ViashDockerPullElseBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "ifneedbepullelsecachedbuild" ]; then + ViashDockerPullElseBuild $image_id $(ViashDockerBuildArgs "$engine_id") + else + ViashError "Unrecognised Docker strategy: $setup_strategy" + exit 1 + fi + elif [ "$setup_strategy" == "push" -o "$setup_strategy" == "forcepush" -o "$setup_strategy" == "alwayspush" ]; then + ViashDockerPush "$image_id" + elif [ "$setup_strategy" == "pushifnotpresent" -o "$setup_strategy" == "gentlepush" -o "$setup_strategy" == "maybepush" ]; then + local save=$-; set +e + ViashDockerRemoteTagCheck $image_id + local outCheck=$? + [[ $save =~ e ]] && set -e + if [ $outCheck -eq 0 ]; then + ViashNotice "Container '$image_id' exists, doing nothing." + else + ViashNotice "Container '$image_id' does not yet exist." + ViashDockerPush "$image_id" + fi + elif [ "$setup_strategy" == "donothing" -o "$setup_strategy" == "meh" ]; then + ViashNotice "Skipping setup." + else + ViashError "Unrecognised Docker strategy: $setup_strategy" + exit 1 + fi +} + +# ViashDockerCheckCommands: Check whether a docker container has the required commands +# +# $1 : image identifier with format `[registry/]image[:tag]` +# $@ : commands to verify being present +# examples: +# ViashDockerCheckCommands bash:4.0 bash ps foo +function ViashDockerCheckCommands { + local image_id="$1" + shift 1 + local commands="$@" + local save=$-; set +e + local missing # mark 'missing' as local in advance, otherwise the exit code of the command will be missing and always be '0' + missing=$(docker run --rm --entrypoint=sh "$image_id" -c "for command in $commands; do command -v \$command >/dev/null 2>&1; if [ \$? -ne 0 ]; then echo \$command; exit 1; fi; done") + local outCheck=$? + [[ $save =~ e ]] && set -e + if [ $outCheck -ne 0 ]; then + ViashError "Docker container '$image_id' does not contain command '$missing'." + exit 1 + fi +} + +# ViashDockerBuild: build a docker image +# $1 : image identifier with format `[registry/]image[:tag]` +# $... : additional arguments to pass to docker build +# $VIASH_META_TEMP_DIR : temporary directory to store dockerfile & optional resources in +# $VIASH_META_NAME : name of the component +# $VIASH_META_RESOURCES_DIR : directory containing the resources +# $VIASH_VERBOSITY : verbosity level +# exit code $? : whether or not the image was built successfully +function ViashDockerBuild { + local image_id="$1" + shift 1 + + # create temporary directory to store dockerfile & optional resources in + local tmpdir=$(mktemp -d "$VIASH_META_TEMP_DIR/dockerbuild-$VIASH_META_NAME-XXXXXX") + local dockerfile="$tmpdir/Dockerfile" + function clean_up { + rm -rf "$tmpdir" + } + trap clean_up EXIT + + # store dockerfile and resources + ViashDockerfile "$VIASH_ENGINE_ID" > "$dockerfile" + + # generate the build command + local docker_build_cmd="docker build -t '$image_id' $@ '$VIASH_META_RESOURCES_DIR' -f '$dockerfile'" + + # build the container + ViashNotice "Building container '$image_id' with Dockerfile" + ViashInfo "$docker_build_cmd" + local save=$-; set +e + if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then + eval $docker_build_cmd + else + eval $docker_build_cmd &> "$tmpdir/docker_build.log" + fi + + # check exit code + local out=$? + [[ $save =~ e ]] && set -e + if [ $out -ne 0 ]; then + ViashError "Error occurred while building container '$image_id'" + if [ $VIASH_VERBOSITY -lt $VIASH_LOGCODE_INFO ]; then + ViashError "Transcript: --------------------------------" + cat "$tmpdir/docker_build.log" + ViashError "End of transcript --------------------------" + fi + exit 1 + fi +} + +######## End of helper functions for setting up Docker images for viash ######## + +# ViashDockerFile: print the dockerfile to stdout +# $1 : engine identifier +# return : dockerfile required to run this component +# examples: +# ViashDockerFile +function ViashDockerfile { + local engine_id="$1" + + if [[ "$engine_id" == "docker" ]]; then + cat << 'VIASHDOCKER' +FROM debian:stable-slim +ENTRYPOINT [] +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y bedtools procps && \ + rm -rf /var/lib/apt/lists/* + +RUN echo "bedtools: \"$(bedtools --version | sed -n 's/^bedtools //p')\"" > /var/software_versions.txt + +LABEL org.opencontainers.image.authors="Theodoro Gasperin Terra Camargo" +LABEL org.opencontainers.image.description="Companion container for running component bedtools bedtools_links" +LABEL org.opencontainers.image.created="2024-09-02T13:02:38Z" +LABEL org.opencontainers.image.source="https://github.com/arq5x/bedtools2" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" +LABEL org.opencontainers.image.version="main" + +VIASHDOCKER + fi +} + +# ViashDockerBuildArgs: return the arguments to pass to docker build +# $1 : engine identifier +# return : arguments to pass to docker build +function ViashDockerBuildArgs { + local engine_id="$1" + + if [[ "$engine_id" == "docker" ]]; then + echo "" + fi +} + +# ViashAbsolutePath: generate absolute path from relative path +# borrowed from https://stackoverflow.com/a/21951256 +# $1 : relative filename +# return : absolute path +# examples: +# ViashAbsolutePath some_file.txt # returns /path/to/some_file.txt +# ViashAbsolutePath /foo/bar/.. # returns /foo +function ViashAbsolutePath { + local thePath + local parr + local outp + local len + if [[ ! "$1" =~ ^/ ]]; then + thePath="$PWD/$1" + else + thePath="$1" + fi + echo "$thePath" | ( + IFS=/ + read -a parr + declare -a outp + for i in "${parr[@]}"; do + case "$i" in + ''|.) continue ;; + ..) + len=${#outp[@]} + if ((len==0)); then + continue + else + unset outp[$((len-1))] + fi + ;; + *) + len=${#outp[@]} + outp[$len]="$i" + ;; + esac + done + echo /"${outp[*]}" + ) +} +# ViashDockerAutodetectMount: auto configuring docker mounts from parameters +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts +# examples: +# ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' +# ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' +function ViashDockerAutodetectMount { + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name + if [ -d "$abs_path" ]; then + mount_source="$abs_path" + base_name="" + else + mount_source=`dirname "$abs_path"` + base_name=`basename "$abs_path"` + fi + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" + if [ -z "$base_name" ]; then + echo "$mount_target" + else + echo "$mount_target/$base_name" + fi +} +function ViashDockerAutodetectMountArg { + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name + if [ -d "$abs_path" ]; then + mount_source="$abs_path" + base_name="" + else + mount_source=`dirname "$abs_path"` + base_name=`basename "$abs_path"` + fi + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" + ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" + echo "--volume=\"$mount_source:$mount_target\"" +} +function ViashDockerStripAutomount { + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" +} +# initialise variables +VIASH_DIRECTORY_MOUNTS=() + +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + +# initialise docker variables +VIASH_DOCKER_RUN_ARGS=(-i --rm) + +# initialise array +VIASH_POSITIONAL_ARGS='' + +while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) + ViashHelp + exit + ;; + ---v|---verbose) + let "VIASH_VERBOSITY=VIASH_VERBOSITY+1" + shift 1 + ;; + ---verbosity) + VIASH_VERBOSITY="$2" + shift 2 + ;; + ---verbosity=*) + VIASH_VERBOSITY="$(ViashRemoveFlags "$1")" + shift 1 + ;; + --version) + echo "bedtools_links main" + exit + ;; + --input) + [ -n "$VIASH_PAR_INPUT" ] && ViashError Bad arguments for option \'--input\': \'$VIASH_PAR_INPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_INPUT="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --input. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --input=*) + [ -n "$VIASH_PAR_INPUT" ] && ViashError Bad arguments for option \'--input=*\': \'$VIASH_PAR_INPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_INPUT=$(ViashRemoveFlags "$1") + shift 1 + ;; + -i) + [ -n "$VIASH_PAR_INPUT" ] && ViashError Bad arguments for option \'-i\': \'$VIASH_PAR_INPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_INPUT="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to -i. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --output) + [ -n "$VIASH_PAR_OUTPUT" ] && ViashError Bad arguments for option \'--output\': \'$VIASH_PAR_OUTPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_OUTPUT="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --output. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --output=*) + [ -n "$VIASH_PAR_OUTPUT" ] && ViashError Bad arguments for option \'--output=*\': \'$VIASH_PAR_OUTPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_OUTPUT=$(ViashRemoveFlags "$1") + shift 1 + ;; + -o) + [ -n "$VIASH_PAR_OUTPUT" ] && ViashError Bad arguments for option \'-o\': \'$VIASH_PAR_OUTPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_OUTPUT="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to -o. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --base_url) + [ -n "$VIASH_PAR_BASE_URL" ] && ViashError Bad arguments for option \'--base_url\': \'$VIASH_PAR_BASE_URL\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_BASE_URL="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --base_url. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --base_url=*) + [ -n "$VIASH_PAR_BASE_URL" ] && ViashError Bad arguments for option \'--base_url=*\': \'$VIASH_PAR_BASE_URL\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_BASE_URL=$(ViashRemoveFlags "$1") + shift 1 + ;; + -base) + [ -n "$VIASH_PAR_BASE_URL" ] && ViashError Bad arguments for option \'-base\': \'$VIASH_PAR_BASE_URL\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_BASE_URL="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to -base. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --organism) + [ -n "$VIASH_PAR_ORGANISM" ] && ViashError Bad arguments for option \'--organism\': \'$VIASH_PAR_ORGANISM\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_ORGANISM="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --organism. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --organism=*) + [ -n "$VIASH_PAR_ORGANISM" ] && ViashError Bad arguments for option \'--organism=*\': \'$VIASH_PAR_ORGANISM\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_ORGANISM=$(ViashRemoveFlags "$1") + shift 1 + ;; + -org) + [ -n "$VIASH_PAR_ORGANISM" ] && ViashError Bad arguments for option \'-org\': \'$VIASH_PAR_ORGANISM\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_ORGANISM="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to -org. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --database) + [ -n "$VIASH_PAR_DATABASE" ] && ViashError Bad arguments for option \'--database\': \'$VIASH_PAR_DATABASE\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_DATABASE="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --database. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --database=*) + [ -n "$VIASH_PAR_DATABASE" ] && ViashError Bad arguments for option \'--database=*\': \'$VIASH_PAR_DATABASE\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_DATABASE=$(ViashRemoveFlags "$1") + shift 1 + ;; + -db) + [ -n "$VIASH_PAR_DATABASE" ] && ViashError Bad arguments for option \'-db\': \'$VIASH_PAR_DATABASE\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_DATABASE="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to -db. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + ---engine) + VIASH_ENGINE_ID="$2" + shift 2 + ;; + ---engine=*) + VIASH_ENGINE_ID="$(ViashRemoveFlags "$1")" + shift 1 + ;; + ---setup) + VIASH_MODE='setup' + VIASH_SETUP_STRATEGY="$2" + shift 2 + ;; + ---setup=*) + VIASH_MODE='setup' + VIASH_SETUP_STRATEGY="$(ViashRemoveFlags "$1")" + shift 1 + ;; + ---dockerfile) + VIASH_MODE='dockerfile' + shift 1 + ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; + ---debug) + VIASH_MODE='debug' + shift 1 + ;; + ---cpus) + [ -n "$VIASH_META_CPUS" ] && ViashError Bad arguments for option \'---cpus\': \'$VIASH_META_CPUS\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_META_CPUS="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to ---cpus. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + ---cpus=*) + [ -n "$VIASH_META_CPUS" ] && ViashError Bad arguments for option \'---cpus=*\': \'$VIASH_META_CPUS\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_META_CPUS=$(ViashRemoveFlags "$1") + shift 1 + ;; + ---memory) + [ -n "$VIASH_META_MEMORY" ] && ViashError Bad arguments for option \'---memory\': \'$VIASH_META_MEMORY\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_META_MEMORY="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to ---memory. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + ---memory=*) + [ -n "$VIASH_META_MEMORY" ] && ViashError Bad arguments for option \'---memory=*\': \'$VIASH_META_MEMORY\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_META_MEMORY=$(ViashRemoveFlags "$1") + shift 1 + ;; + *) # positional arg or unknown option + # since the positional args will be eval'd, can we always quote, instead of using ViashQuote + VIASH_POSITIONAL_ARGS="$VIASH_POSITIONAL_ARGS '$1'" + [[ $1 == -* ]] && ViashWarning $1 looks like a parameter but is not a defined parameter and will instead be treated as a positional argument. Use "--help" to get more information on the parameters. + shift # past argument + ;; + esac +done + +# parse positional parameters +eval set -- $VIASH_POSITIONAL_ARGS + + +if [ "$VIASH_ENGINE_ID" == "native" ] ; then + VIASH_ENGINE_TYPE='native' +elif [ "$VIASH_ENGINE_ID" == "docker" ] ; then + VIASH_ENGINE_TYPE='docker' +else + ViashError "Engine '$VIASH_ENGINE_ID' is not recognized. Options are: docker, native." + exit 1 +fi + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + # check if docker is installed properly + ViashDockerInstallationCheck + + # determine docker image id + if [[ "$VIASH_ENGINE_ID" == 'docker' ]]; then + VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/biobox/bedtools/bedtools_links:main' + fi + + # print dockerfile + if [ "$VIASH_MODE" == "dockerfile" ]; then + ViashDockerfile "$VIASH_ENGINE_ID" + exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 + + # enter docker container + elif [[ "$VIASH_MODE" == "debug" ]]; then + VIASH_CMD="docker run --entrypoint=bash ${VIASH_DOCKER_RUN_ARGS[@]} -v '$(pwd)':/pwd --workdir /pwd -t $VIASH_DOCKER_IMAGE_ID" + ViashNotice "+ $VIASH_CMD" + eval $VIASH_CMD + exit + + # build docker image + elif [ "$VIASH_MODE" == "setup" ]; then + ViashDockerSetup "$VIASH_DOCKER_IMAGE_ID" "$VIASH_SETUP_STRATEGY" + ViashDockerCheckCommands "$VIASH_DOCKER_IMAGE_ID" 'ps' 'bash' + exit 0 + fi + + # check if docker image exists + ViashDockerSetup "$VIASH_DOCKER_IMAGE_ID" ifneedbepullelsecachedbuild + ViashDockerCheckCommands "$VIASH_DOCKER_IMAGE_ID" 'ps' 'bash' +fi + +# setting computational defaults + +# helper function for parsing memory strings +function ViashMemoryAsBytes { + local memory=`echo "$1" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]'` + local memory_regex='^([0-9]+)([kmgtp]i?b?|b)$' + if [[ $memory =~ $memory_regex ]]; then + local number=${memory/[^0-9]*/} + local symbol=${memory/*[0-9]/} + + case $symbol in + b) memory_b=$number ;; + kb|k) memory_b=$(( $number * 1000 )) ;; + mb|m) memory_b=$(( $number * 1000 * 1000 )) ;; + gb|g) memory_b=$(( $number * 1000 * 1000 * 1000 )) ;; + tb|t) memory_b=$(( $number * 1000 * 1000 * 1000 * 1000 )) ;; + pb|p) memory_b=$(( $number * 1000 * 1000 * 1000 * 1000 * 1000 )) ;; + kib|ki) memory_b=$(( $number * 1024 )) ;; + mib|mi) memory_b=$(( $number * 1024 * 1024 )) ;; + gib|gi) memory_b=$(( $number * 1024 * 1024 * 1024 )) ;; + tib|ti) memory_b=$(( $number * 1024 * 1024 * 1024 * 1024 )) ;; + pib|pi) memory_b=$(( $number * 1024 * 1024 * 1024 * 1024 * 1024 )) ;; + esac + echo "$memory_b" + fi +} +# compute memory in different units +if [ ! -z ${VIASH_META_MEMORY+x} ]; then + VIASH_META_MEMORY_B=`ViashMemoryAsBytes $VIASH_META_MEMORY` + # do not define other variables if memory_b is an empty string + if [ ! -z "$VIASH_META_MEMORY_B" ]; then + VIASH_META_MEMORY_KB=$(( ($VIASH_META_MEMORY_B+999) / 1000 )) + VIASH_META_MEMORY_MB=$(( ($VIASH_META_MEMORY_KB+999) / 1000 )) + VIASH_META_MEMORY_GB=$(( ($VIASH_META_MEMORY_MB+999) / 1000 )) + VIASH_META_MEMORY_TB=$(( ($VIASH_META_MEMORY_GB+999) / 1000 )) + VIASH_META_MEMORY_PB=$(( ($VIASH_META_MEMORY_TB+999) / 1000 )) + VIASH_META_MEMORY_KIB=$(( ($VIASH_META_MEMORY_B+1023) / 1024 )) + VIASH_META_MEMORY_MIB=$(( ($VIASH_META_MEMORY_KIB+1023) / 1024 )) + VIASH_META_MEMORY_GIB=$(( ($VIASH_META_MEMORY_MIB+1023) / 1024 )) + VIASH_META_MEMORY_TIB=$(( ($VIASH_META_MEMORY_GIB+1023) / 1024 )) + VIASH_META_MEMORY_PIB=$(( ($VIASH_META_MEMORY_TIB+1023) / 1024 )) + else + # unset memory if string is empty + unset $VIASH_META_MEMORY_B + fi +fi +# unset nproc if string is empty +if [ -z "$VIASH_META_CPUS" ]; then + unset $VIASH_META_CPUS +fi + + +# check whether required parameters exist +if [ -z ${VIASH_PAR_INPUT+x} ]; then + ViashError '--input' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_NAME+x} ]; then + ViashError 'name' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_FUNCTIONALITY_NAME+x} ]; then + ViashError 'functionality_name' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_RESOURCES_DIR+x} ]; then + ViashError 'resources_dir' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_EXECUTABLE+x} ]; then + ViashError 'executable' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_CONFIG+x} ]; then + ViashError 'config' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_TEMP_DIR+x} ]; then + ViashError 'temp_dir' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi + +# filling in defaults +if [ -z ${VIASH_PAR_BASE_URL+x} ]; then + VIASH_PAR_BASE_URL="http://genome.ucsc.edu" +fi +if [ -z ${VIASH_PAR_ORGANISM+x} ]; then + VIASH_PAR_ORGANISM="human" +fi +if [ -z ${VIASH_PAR_DATABASE+x} ]; then + VIASH_PAR_DATABASE="hg18" +fi + +# check whether required files exist +if [ ! -z "$VIASH_PAR_INPUT" ] && [ ! -e "$VIASH_PAR_INPUT" ]; then + ViashError "Input file '$VIASH_PAR_INPUT' does not exist." + exit 1 +fi + +# check whether parameters values are of the right type +if [[ -n "$VIASH_META_CPUS" ]]; then + if ! [[ "$VIASH_META_CPUS" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'cpus' has to be an integer. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_B" ]]; then + if ! [[ "$VIASH_META_MEMORY_B" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_b' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_KB" ]]; then + if ! [[ "$VIASH_META_MEMORY_KB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_kb' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_MB" ]]; then + if ! [[ "$VIASH_META_MEMORY_MB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_mb' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_GB" ]]; then + if ! [[ "$VIASH_META_MEMORY_GB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_gb' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_TB" ]]; then + if ! [[ "$VIASH_META_MEMORY_TB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_tb' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_PB" ]]; then + if ! [[ "$VIASH_META_MEMORY_PB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_pb' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_KIB" ]]; then + if ! [[ "$VIASH_META_MEMORY_KIB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_kib' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_MIB" ]]; then + if ! [[ "$VIASH_META_MEMORY_MIB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_mib' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_GIB" ]]; then + if ! [[ "$VIASH_META_MEMORY_GIB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_gib' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_TIB" ]]; then + if ! [[ "$VIASH_META_MEMORY_TIB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_tib' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_PIB" ]]; then + if ! [[ "$VIASH_META_MEMORY_PIB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_pib' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi + +# create parent directories of output files, if so desired +if [ ! -z "$VIASH_PAR_OUTPUT" ] && [ ! -d "$(dirname "$VIASH_PAR_OUTPUT")" ]; then + mkdir -p "$(dirname "$VIASH_PAR_OUTPUT")" +fi + +if [ "$VIASH_ENGINE_ID" == "native" ] ; then + if [ "$VIASH_MODE" == "run" ]; then + VIASH_CMD="bash" + else + ViashError "Engine '$VIASH_ENGINE_ID' does not support mode '$VIASH_MODE'." + exit 1 + fi +fi + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + # detect volumes from file arguments + VIASH_CHOWN_VARS=() +if [ ! -z "$VIASH_PAR_INPUT" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_PAR_INPUT")" ) + VIASH_PAR_INPUT=$(ViashDockerAutodetectMount "$VIASH_PAR_INPUT") +fi +if [ ! -z "$VIASH_PAR_OUTPUT" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_PAR_OUTPUT")" ) + VIASH_PAR_OUTPUT=$(ViashDockerAutodetectMount "$VIASH_PAR_OUTPUT") + VIASH_CHOWN_VARS+=( "$VIASH_PAR_OUTPUT" ) +fi +if [ ! -z "$VIASH_META_RESOURCES_DIR" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_RESOURCES_DIR")" ) + VIASH_META_RESOURCES_DIR=$(ViashDockerAutodetectMount "$VIASH_META_RESOURCES_DIR") +fi +if [ ! -z "$VIASH_META_EXECUTABLE" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_EXECUTABLE")" ) + VIASH_META_EXECUTABLE=$(ViashDockerAutodetectMount "$VIASH_META_EXECUTABLE") +fi +if [ ! -z "$VIASH_META_CONFIG" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_CONFIG")" ) + VIASH_META_CONFIG=$(ViashDockerAutodetectMount "$VIASH_META_CONFIG") +fi +if [ ! -z "$VIASH_META_TEMP_DIR" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_TEMP_DIR")" ) + VIASH_META_TEMP_DIR=$(ViashDockerAutodetectMount "$VIASH_META_TEMP_DIR") +fi + + # get unique mounts + VIASH_UNIQUE_MOUNTS=($(for val in "${VIASH_DIRECTORY_MOUNTS[@]}"; do echo "$val"; done | sort -u)) +fi + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + # change file ownership + function ViashPerformChown { + if (( ${#VIASH_CHOWN_VARS[@]} )); then + set +e + VIASH_CMD="docker run --entrypoint=bash --rm ${VIASH_UNIQUE_MOUNTS[@]} $VIASH_DOCKER_IMAGE_ID -c 'chown $(id -u):$(id -g) --silent --recursive ${VIASH_CHOWN_VARS[@]}'" + ViashDebug "+ $VIASH_CMD" + eval $VIASH_CMD + set -e + fi + } + trap ViashPerformChown EXIT +fi + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + # helper function for filling in extra docker args + if [ ! -z "$VIASH_META_MEMORY_B" ]; then + VIASH_DOCKER_RUN_ARGS+=("--memory=${VIASH_META_MEMORY_B}") + fi + if [ ! -z "$VIASH_META_CPUS" ]; then + VIASH_DOCKER_RUN_ARGS+=("--cpus=${VIASH_META_CPUS}") + fi +fi + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + VIASH_CMD="docker run --entrypoint=bash ${VIASH_DOCKER_RUN_ARGS[@]} ${VIASH_UNIQUE_MOUNTS[@]} $VIASH_DOCKER_IMAGE_ID" +fi + + +# set dependency paths + + +ViashDebug "Running command: $(echo $VIASH_CMD)" +cat << VIASHEOF | eval $VIASH_CMD +set -e +tempscript=\$(mktemp "$VIASH_META_TEMP_DIR/viash-run-bedtools_links-XXXXXX").sh +function clean_up { + rm "\$tempscript" +} +function interrupt { + echo -e "\nCTRL-C Pressed..." + exit 1 +} +trap clean_up EXIT +trap interrupt INT SIGINT +cat > "\$tempscript" << 'VIASHMAIN' +#!/bin/bash + +## VIASH START +# The following code has been auto-generated by Viash. +$( if [ ! -z ${VIASH_PAR_INPUT+x} ]; then echo "${VIASH_PAR_INPUT}" | sed "s#'#'\"'\"'#g;s#.*#par_input='&'#" ; else echo "# par_input="; fi ) +$( if [ ! -z ${VIASH_PAR_OUTPUT+x} ]; then echo "${VIASH_PAR_OUTPUT}" | sed "s#'#'\"'\"'#g;s#.*#par_output='&'#" ; else echo "# par_output="; fi ) +$( if [ ! -z ${VIASH_PAR_BASE_URL+x} ]; then echo "${VIASH_PAR_BASE_URL}" | sed "s#'#'\"'\"'#g;s#.*#par_base_url='&'#" ; else echo "# par_base_url="; fi ) +$( if [ ! -z ${VIASH_PAR_ORGANISM+x} ]; then echo "${VIASH_PAR_ORGANISM}" | sed "s#'#'\"'\"'#g;s#.*#par_organism='&'#" ; else echo "# par_organism="; fi ) +$( if [ ! -z ${VIASH_PAR_DATABASE+x} ]; then echo "${VIASH_PAR_DATABASE}" | sed "s#'#'\"'\"'#g;s#.*#par_database='&'#" ; else echo "# par_database="; fi ) +$( if [ ! -z ${VIASH_META_NAME+x} ]; then echo "${VIASH_META_NAME}" | sed "s#'#'\"'\"'#g;s#.*#meta_name='&'#" ; else echo "# meta_name="; fi ) +$( if [ ! -z ${VIASH_META_FUNCTIONALITY_NAME+x} ]; then echo "${VIASH_META_FUNCTIONALITY_NAME}" | sed "s#'#'\"'\"'#g;s#.*#meta_functionality_name='&'#" ; else echo "# meta_functionality_name="; fi ) +$( if [ ! -z ${VIASH_META_RESOURCES_DIR+x} ]; then echo "${VIASH_META_RESOURCES_DIR}" | sed "s#'#'\"'\"'#g;s#.*#meta_resources_dir='&'#" ; else echo "# meta_resources_dir="; fi ) +$( if [ ! -z ${VIASH_META_EXECUTABLE+x} ]; then echo "${VIASH_META_EXECUTABLE}" | sed "s#'#'\"'\"'#g;s#.*#meta_executable='&'#" ; else echo "# meta_executable="; fi ) +$( if [ ! -z ${VIASH_META_CONFIG+x} ]; then echo "${VIASH_META_CONFIG}" | sed "s#'#'\"'\"'#g;s#.*#meta_config='&'#" ; else echo "# meta_config="; fi ) +$( if [ ! -z ${VIASH_META_TEMP_DIR+x} ]; then echo "${VIASH_META_TEMP_DIR}" | sed "s#'#'\"'\"'#g;s#.*#meta_temp_dir='&'#" ; else echo "# meta_temp_dir="; fi ) +$( if [ ! -z ${VIASH_META_CPUS+x} ]; then echo "${VIASH_META_CPUS}" | sed "s#'#'\"'\"'#g;s#.*#meta_cpus='&'#" ; else echo "# meta_cpus="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_B+x} ]; then echo "${VIASH_META_MEMORY_B}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_b='&'#" ; else echo "# meta_memory_b="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_KB+x} ]; then echo "${VIASH_META_MEMORY_KB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_kb='&'#" ; else echo "# meta_memory_kb="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_MB+x} ]; then echo "${VIASH_META_MEMORY_MB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_mb='&'#" ; else echo "# meta_memory_mb="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_GB+x} ]; then echo "${VIASH_META_MEMORY_GB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_gb='&'#" ; else echo "# meta_memory_gb="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_TB+x} ]; then echo "${VIASH_META_MEMORY_TB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_tb='&'#" ; else echo "# meta_memory_tb="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_PB+x} ]; then echo "${VIASH_META_MEMORY_PB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_pb='&'#" ; else echo "# meta_memory_pb="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_KIB+x} ]; then echo "${VIASH_META_MEMORY_KIB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_kib='&'#" ; else echo "# meta_memory_kib="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_MIB+x} ]; then echo "${VIASH_META_MEMORY_MIB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_mib='&'#" ; else echo "# meta_memory_mib="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_GIB+x} ]; then echo "${VIASH_META_MEMORY_GIB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_gib='&'#" ; else echo "# meta_memory_gib="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_TIB+x} ]; then echo "${VIASH_META_MEMORY_TIB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_tib='&'#" ; else echo "# meta_memory_tib="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_PIB+x} ]; then echo "${VIASH_META_MEMORY_PIB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_pib='&'#" ; else echo "# meta_memory_pib="; fi ) + +## VIASH END + +set -eo pipefail + +# Execute bedtools links +bedtools links \\ + \${par_base_url:+-base "\$par_base_url"} \\ + \${par_organism:+-org "\$par_organism"} \\ + \${par_database:+-db "\$par_database"} \\ + -i "\$par_input" \\ + > "\$par_output" +VIASHMAIN +bash "\$tempscript" & +wait "\$!" + +VIASHEOF + + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + # strip viash automount from file paths + + if [ ! -z "$VIASH_PAR_INPUT" ]; then + VIASH_PAR_INPUT=$(ViashDockerStripAutomount "$VIASH_PAR_INPUT") + fi + if [ ! -z "$VIASH_PAR_OUTPUT" ]; then + VIASH_PAR_OUTPUT=$(ViashDockerStripAutomount "$VIASH_PAR_OUTPUT") + fi + if [ ! -z "$VIASH_META_RESOURCES_DIR" ]; then + VIASH_META_RESOURCES_DIR=$(ViashDockerStripAutomount "$VIASH_META_RESOURCES_DIR") + fi + if [ ! -z "$VIASH_META_EXECUTABLE" ]; then + VIASH_META_EXECUTABLE=$(ViashDockerStripAutomount "$VIASH_META_EXECUTABLE") + fi + if [ ! -z "$VIASH_META_CONFIG" ]; then + VIASH_META_CONFIG=$(ViashDockerStripAutomount "$VIASH_META_CONFIG") + fi + if [ ! -z "$VIASH_META_TEMP_DIR" ]; then + VIASH_META_TEMP_DIR=$(ViashDockerStripAutomount "$VIASH_META_TEMP_DIR") + fi +fi + + +# check whether required files exist +if [ ! -z "$VIASH_PAR_OUTPUT" ] && [ ! -e "$VIASH_PAR_OUTPUT" ]; then + ViashError "Output file '$VIASH_PAR_OUTPUT' does not exist." + exit 1 +fi + + +exit 0 diff --git a/target/executable/bedtools/bedtools_merge/.config.vsh.yaml b/target/executable/bedtools/bedtools_merge/.config.vsh.yaml new file mode 100644 index 00000000..5094e32e --- /dev/null +++ b/target/executable/bedtools/bedtools_merge/.config.vsh.yaml @@ -0,0 +1,305 @@ +name: "bedtools_merge" +namespace: "bedtools" +version: "main" +authors: +- name: "Theodoro Gasperin Terra Camargo" + roles: + - "author" + - "maintainer" + info: + links: + email: "theodorogtc@gmail.com" + github: "tgaspe" + linkedin: "theodoro-gasperin-terra-camargo" + organizations: + - name: "Data Intuitive" + href: "https://www.data-intuitive.com" + role: "Bioinformatician" +argument_groups: +- name: "Inputs" + arguments: + - type: "file" + name: "--input" + alternatives: + - "-i" + description: "Input file (BED/GFF/VCF) to be merged." + info: null + must_exist: true + create_parent: true + required: true + direction: "input" + multiple: false + multiple_sep: ";" +- name: "Outputs" + arguments: + - type: "file" + name: "--output" + description: "Output merged file BED to be written." + info: null + must_exist: true + create_parent: true + required: true + direction: "output" + multiple: false + multiple_sep: ";" +- name: "Options" + arguments: + - type: "boolean_true" + name: "--strand" + alternatives: + - "-s" + description: "Force strandedness. That is, only merge features\nthat are on the\ + \ same strand.\n- By default, merging is done without respect to strand.\n" + info: null + direction: "input" + - type: "string" + name: "--specific_strand" + alternatives: + - "-S" + description: "Force merge for one specific strand only.\nFollow with + or - to\ + \ force merge from only\nthe forward or reverse strand, respectively.\n- By\ + \ default, merging is done without respect to strand.\n" + info: null + required: false + choices: + - "+" + - "-" + direction: "input" + multiple: false + multiple_sep: ";" + - type: "integer" + name: "--distance" + alternatives: + - "-d" + description: "Maximum distance between features allowed for features\nto be merged.\n\ + - Def. 0. That is, overlapping & book-ended features are merged.\n- (INTEGER)\n\ + - Note: negative values enforce the number of b.p. required for overlap.\n" + info: null + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "integer" + name: "--columns" + alternatives: + - "-c" + description: "Specify columns from the B file to map onto intervals in A.\nDefault:\ + \ 5.\nMultiple columns can be specified in a comma-delimited list.\n" + info: null + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "string" + name: "--operation" + alternatives: + - "-o" + description: "Specify the operation that should be applied to -c.\nValid operations:\n\ + \ sum, min, max, absmin, absmax,\n mean, median, mode, antimode\n stdev,\ + \ sstdev\n collapse (i.e., print a delimited list (duplicates allowed)),\ + \ \n distinct (i.e., print a delimited list (NO duplicates allowed)), \n\ + \ distinct_sort_num (as distinct, sorted numerically, ascending),\n distinct_sort_num_desc\ + \ (as distinct, sorted numerically, desscending),\n distinct_only (delimited\ + \ list of only unique values),\n count\n count_distinct (i.e., a count\ + \ of the unique values in the column), \n first (i.e., just the first value\ + \ in the column), \n last (i.e., just the last value in the column), \nDefault:\ + \ sum\nMultiple operations can be specified in a comma-delimited list.\n\nIf\ + \ there is only column, but multiple operations, all operations will be\napplied\ + \ on that column. Likewise, if there is only one operation, but\nmultiple columns,\ + \ that operation will be applied to all columns.\nOtherwise, the number of columns\ + \ must match the the number of operations,\nand will be applied in respective\ + \ order.\nE.g., \"-c 5,4,6 -o sum,mean,count\" will give the sum of column 5,\n\ + the mean of column 4, and the count of column 6.\nThe order of output columns\ + \ will match the ordering given in the command.\n" + info: null + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "string" + name: "--delimiter" + alternatives: + - "-delim" + description: "Specify a custom delimiter for the collapse operations.\n" + info: null + example: + - "|" + default: + - "," + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "integer" + name: "--precision" + alternatives: + - "-prec" + description: "Sets the decimal precision for output (Default: 5).\n" + info: null + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "boolean_true" + name: "--bed" + description: "If using BAM input, write output as BED.\n" + info: null + direction: "input" + - type: "boolean_true" + name: "--header" + description: "Print the header from the A file prior to results.\n" + info: null + direction: "input" + - type: "boolean_true" + name: "--no_buffer" + alternatives: + - "-nobuf" + description: "Disable buffered output. Using this option will cause each line\n\ + of output to be printed as it is generated, rather than saved\nin a buffer.\ + \ This will make printing large output files \nnoticeably slower, but can be\ + \ useful in conjunction with\nother software tools and scripts that need to\ + \ process one\nline of bedtools output at a time.\n" + info: null + direction: "input" +resources: +- type: "bash_script" + path: "script.sh" + is_executable: true +description: "Merges overlapping BED/GFF/VCF entries into a single interval.\n" +test_resources: +- type: "bash_script" + path: "test.sh" + is_executable: true +- type: "file" + path: "test_data" +info: null +status: "enabled" +requirements: + commands: + - "ps" +license: "MIT" +references: + doi: + - "10.1093/bioinformatics/btq033" +links: + repository: "https://github.com/arq5x/bedtools2" + homepage: "https://bedtools.readthedocs.io/en/latest/#" + documentation: "https://bedtools.readthedocs.io/en/latest/content/tools/merge.html" + issue_tracker: "https://github.com/arq5x/bedtools2/issues" +runners: +- type: "executable" + id: "executable" + docker_setup_strategy: "ifneedbepullelsecachedbuild" +- type: "nextflow" + id: "nextflow" + directives: + tag: "$id" + auto: + simplifyInput: true + simplifyOutput: false + transcript: false + publish: false + config: + labels: + mem1gb: "memory = 1000000000.B" + mem2gb: "memory = 2000000000.B" + mem5gb: "memory = 5000000000.B" + mem10gb: "memory = 10000000000.B" + mem20gb: "memory = 20000000000.B" + mem50gb: "memory = 50000000000.B" + mem100gb: "memory = 100000000000.B" + mem200gb: "memory = 200000000000.B" + mem500gb: "memory = 500000000000.B" + mem1tb: "memory = 1000000000000.B" + mem2tb: "memory = 2000000000000.B" + mem5tb: "memory = 5000000000000.B" + mem10tb: "memory = 10000000000000.B" + mem20tb: "memory = 20000000000000.B" + mem50tb: "memory = 50000000000000.B" + mem100tb: "memory = 100000000000000.B" + mem200tb: "memory = 200000000000000.B" + mem500tb: "memory = 500000000000000.B" + mem1gib: "memory = 1073741824.B" + mem2gib: "memory = 2147483648.B" + mem4gib: "memory = 4294967296.B" + mem8gib: "memory = 8589934592.B" + mem16gib: "memory = 17179869184.B" + mem32gib: "memory = 34359738368.B" + mem64gib: "memory = 68719476736.B" + mem128gib: "memory = 137438953472.B" + mem256gib: "memory = 274877906944.B" + mem512gib: "memory = 549755813888.B" + mem1tib: "memory = 1099511627776.B" + mem2tib: "memory = 2199023255552.B" + mem4tib: "memory = 4398046511104.B" + mem8tib: "memory = 8796093022208.B" + mem16tib: "memory = 17592186044416.B" + mem32tib: "memory = 35184372088832.B" + mem64tib: "memory = 70368744177664.B" + mem128tib: "memory = 140737488355328.B" + mem256tib: "memory = 281474976710656.B" + mem512tib: "memory = 562949953421312.B" + cpu1: "cpus = 1" + cpu2: "cpus = 2" + cpu5: "cpus = 5" + cpu10: "cpus = 10" + cpu20: "cpus = 20" + cpu50: "cpus = 50" + cpu100: "cpus = 100" + cpu200: "cpus = 200" + cpu500: "cpus = 500" + cpu1000: "cpus = 1000" + debug: false + container: "docker" +engines: +- type: "docker" + id: "docker" + image: "debian:stable-slim" + target_registry: "images.viash-hub.com" + target_tag: "main" + namespace_separator: "/" + setup: + - type: "apt" + packages: + - "bedtools" + - "procps" + interactive: false + - type: "docker" + run: + - "echo \"bedtools: \\\"$(bedtools --version | sed -n 's/^bedtools //p')\\\"\"\ + \ > /var/software_versions.txt\n" + entrypoint: [] + cmd: null +- type: "native" + id: "native" +build_info: + config: "src/bedtools/bedtools_merge/config.vsh.yaml" + runner: "executable" + engine: "docker|native" + output: "target/executable/bedtools/bedtools_merge" + executable: "target/executable/bedtools/bedtools_merge/bedtools_merge" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" + git_remote: "https://github.com/viash-hub/biobox" +package_config: + name: "biobox" + version: "main" + description: "A collection of bioinformatics tools for working with sequence data.\n" + info: null + viash_version: "0.9.0-RC7" + source: "src" + target: "target" + config_mods: + - ".requirements.commands := ['ps']\n" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'main'" + keywords: + - "bioinformatics" + - "modules" + - "sequencing" + license: "MIT" + organization: "vsh" + links: + repository: "https://github.com/viash-hub/biobox" + issue_tracker: "https://github.com/viash-hub/biobox/issues" diff --git a/target/executable/bedtools/bedtools_merge/bedtools_merge b/target/executable/bedtools/bedtools_merge/bedtools_merge new file mode 100755 index 00000000..7172c61e --- /dev/null +++ b/target/executable/bedtools/bedtools_merge/bedtools_merge @@ -0,0 +1,1418 @@ +#!/usr/bin/env bash + +# bedtools_merge main +# +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a +# derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from +# Data Intuitive. +# +# The component may contain files which fall under a different license. The +# authors of this component should specify the license in the header of such +# files, or include a separate license file detailing the licenses of all included +# files. +# +# Component authors: +# * Theodoro Gasperin Terra Camargo (author, maintainer) + +set -e + +if [ -z "$VIASH_TEMP" ]; then + VIASH_TEMP=${VIASH_TEMP:-$VIASH_TMPDIR} + VIASH_TEMP=${VIASH_TEMP:-$VIASH_TEMPDIR} + VIASH_TEMP=${VIASH_TEMP:-$VIASH_TMP} + VIASH_TEMP=${VIASH_TEMP:-$TMPDIR} + VIASH_TEMP=${VIASH_TEMP:-$TMP} + VIASH_TEMP=${VIASH_TEMP:-$TEMPDIR} + VIASH_TEMP=${VIASH_TEMP:-$TEMP} + VIASH_TEMP=${VIASH_TEMP:-/tmp} +fi + +# define helper functions +# ViashQuote: put quotes around non flag values +# $1 : unquoted string +# return : possibly quoted string +# examples: +# ViashQuote --foo # returns --foo +# ViashQuote bar # returns 'bar' +# Viashquote --foo=bar # returns --foo='bar' +function ViashQuote { + if [[ "$1" =~ ^-+[a-zA-Z0-9_\-]+=.+$ ]]; then + echo "$1" | sed "s#=\(.*\)#='\1'#" + elif [[ "$1" =~ ^-+[a-zA-Z0-9_\-]+$ ]]; then + echo "$1" + else + echo "'$1'" + fi +} +# ViashRemoveFlags: Remove leading flag +# $1 : string with a possible leading flag +# return : string without possible leading flag +# examples: +# ViashRemoveFlags --foo=bar # returns bar +function ViashRemoveFlags { + echo "$1" | sed 's/^--*[a-zA-Z0-9_\-]*=//' +} +# ViashSourceDir: return the path of a bash file, following symlinks +# usage : ViashSourceDir ${BASH_SOURCE[0]} +# $1 : Should always be set to ${BASH_SOURCE[0]} +# returns : The absolute path of the bash file +function ViashSourceDir { + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" + done + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd +} +# ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks +# usage : ViashFindTargetDir 'ScriptPath' +# $1 : The location from where to start the upward search +# returns : The absolute path of the '.build.yaml' file +function ViashFindTargetDir { + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} + done + echo $source +} +# see https://en.wikipedia.org/wiki/Syslog#Severity_level +VIASH_LOGCODE_EMERGENCY=0 +VIASH_LOGCODE_ALERT=1 +VIASH_LOGCODE_CRITICAL=2 +VIASH_LOGCODE_ERROR=3 +VIASH_LOGCODE_WARNING=4 +VIASH_LOGCODE_NOTICE=5 +VIASH_LOGCODE_INFO=6 +VIASH_LOGCODE_DEBUG=7 +VIASH_VERBOSITY=$VIASH_LOGCODE_NOTICE + +# ViashLog: Log events depending on the verbosity level +# usage: ViashLog 1 alert Oh no something went wrong! +# $1: required verbosity level +# $2: display tag +# $3+: messages to display +# stdout: Your input, prepended by '[$2] '. +function ViashLog { + local required_level="$1" + local display_tag="$2" + shift 2 + if [ $VIASH_VERBOSITY -ge $required_level ]; then + >&2 echo "[$display_tag]" "$@" + fi +} + +# ViashEmergency: log events when the system is unstable +# usage: ViashEmergency Oh no something went wrong. +# stdout: Your input, prepended by '[emergency] '. +function ViashEmergency { + ViashLog $VIASH_LOGCODE_EMERGENCY emergency "$@" +} + +# ViashAlert: log events when actions must be taken immediately (e.g. corrupted system database) +# usage: ViashAlert Oh no something went wrong. +# stdout: Your input, prepended by '[alert] '. +function ViashAlert { + ViashLog $VIASH_LOGCODE_ALERT alert "$@" +} + +# ViashCritical: log events when a critical condition occurs +# usage: ViashCritical Oh no something went wrong. +# stdout: Your input, prepended by '[critical] '. +function ViashCritical { + ViashLog $VIASH_LOGCODE_CRITICAL critical "$@" +} + +# ViashError: log events when an error condition occurs +# usage: ViashError Oh no something went wrong. +# stdout: Your input, prepended by '[error] '. +function ViashError { + ViashLog $VIASH_LOGCODE_ERROR error "$@" +} + +# ViashWarning: log potentially abnormal events +# usage: ViashWarning Something may have gone wrong. +# stdout: Your input, prepended by '[warning] '. +function ViashWarning { + ViashLog $VIASH_LOGCODE_WARNING warning "$@" +} + +# ViashNotice: log significant but normal events +# usage: ViashNotice This just happened. +# stdout: Your input, prepended by '[notice] '. +function ViashNotice { + ViashLog $VIASH_LOGCODE_NOTICE notice "$@" +} + +# ViashInfo: log normal events +# usage: ViashInfo This just happened. +# stdout: Your input, prepended by '[info] '. +function ViashInfo { + ViashLog $VIASH_LOGCODE_INFO info "$@" +} + +# ViashDebug: log all events, for debugging purposes +# usage: ViashDebug This just happened. +# stdout: Your input, prepended by '[debug] '. +function ViashDebug { + ViashLog $VIASH_LOGCODE_DEBUG debug "$@" +} + +# find source folder of this component +VIASH_META_RESOURCES_DIR=`ViashSourceDir ${BASH_SOURCE[0]}` + +# find the root of the built components & dependencies +VIASH_TARGET_DIR=`ViashFindTargetDir $VIASH_META_RESOURCES_DIR` + +# define meta fields +VIASH_META_NAME="bedtools_merge" +VIASH_META_FUNCTIONALITY_NAME="bedtools_merge" +VIASH_META_EXECUTABLE="$VIASH_META_RESOURCES_DIR/$VIASH_META_NAME" +VIASH_META_CONFIG="$VIASH_META_RESOURCES_DIR/.config.vsh.yaml" +VIASH_META_TEMP_DIR="$VIASH_TEMP" + + +# ViashHelp: Display helpful explanation about this executable +function ViashHelp { + echo "bedtools_merge main" + echo "" + echo "Merges overlapping BED/GFF/VCF entries into a single interval." + echo "" + echo "Inputs:" + echo " -i, --input" + echo " type: file, required parameter, file must exist" + echo " Input file (BED/GFF/VCF) to be merged." + echo "" + echo "Outputs:" + echo " --output" + echo " type: file, required parameter, output, file must exist" + echo " Output merged file BED to be written." + echo "" + echo "Options:" + echo " -s, --strand" + echo " type: boolean_true" + echo " Force strandedness. That is, only merge features" + echo " that are on the same strand." + echo " - By default, merging is done without respect to strand." + echo "" + echo " -S, --specific_strand" + echo " type: string" + echo " choices: [ +, - ]" + echo " Force merge for one specific strand only." + echo " Follow with + or - to force merge from only" + echo " the forward or reverse strand, respectively." + echo " - By default, merging is done without respect to strand." + echo "" + echo " -d, --distance" + echo " type: integer" + echo " Maximum distance between features allowed for features" + echo " to be merged." + echo " - Def. 0. That is, overlapping & book-ended features are merged." + echo " - (INTEGER)" + echo " - Note: negative values enforce the number of b.p. required for overlap." + echo "" + echo " -c, --columns" + echo " type: integer" + echo " Specify columns from the B file to map onto intervals in A." + echo " Default: 5." + echo " Multiple columns can be specified in a comma-delimited list." + echo "" + echo " -o, --operation" + echo " type: string" + echo " Specify the operation that should be applied to -c." + echo " Valid operations:" + echo " sum, min, max, absmin, absmax," + echo " mean, median, mode, antimode" + echo " stdev, sstdev" + echo " collapse (i.e., print a delimited list (duplicates allowed))," + echo " distinct (i.e., print a delimited list (NO duplicates allowed))," + echo " distinct_sort_num (as distinct, sorted numerically, ascending)," + echo " distinct_sort_num_desc (as distinct, sorted numerically," + echo " desscending)," + echo " distinct_only (delimited list of only unique values)," + echo " count" + echo " count_distinct (i.e., a count of the unique values in the column)," + echo " first (i.e., just the first value in the column)," + echo " last (i.e., just the last value in the column)," + echo " Default: sum" + echo " Multiple operations can be specified in a comma-delimited list." + echo " If there is only column, but multiple operations, all operations will be" + echo " applied on that column. Likewise, if there is only one operation, but" + echo " multiple columns, that operation will be applied to all columns." + echo " Otherwise, the number of columns must match the the number of" + echo " operations," + echo " and will be applied in respective order." + echo " E.g., \"-c 5,4,6 -o sum,mean,count\" will give the sum of column 5," + echo " the mean of column 4, and the count of column 6." + echo " The order of output columns will match the ordering given in the" + echo " command." + echo "" + echo " -delim, --delimiter" + echo " type: string" + echo " default: ," + echo " example: |" + echo " Specify a custom delimiter for the collapse operations." + echo "" + echo " -prec, --precision" + echo " type: integer" + echo " Sets the decimal precision for output (Default: 5)." + echo "" + echo " --bed" + echo " type: boolean_true" + echo " If using BAM input, write output as BED." + echo "" + echo " --header" + echo " type: boolean_true" + echo " Print the header from the A file prior to results." + echo "" + echo " -nobuf, --no_buffer" + echo " type: boolean_true" + echo " Disable buffered output. Using this option will cause each line" + echo " of output to be printed as it is generated, rather than saved" + echo " in a buffer. This will make printing large output files" + echo " noticeably slower, but can be useful in conjunction with" + echo " other software tools and scripts that need to process one" + echo " line of bedtools output at a time." +} + +# initialise variables +VIASH_MODE='run' +VIASH_ENGINE_ID='docker' + +######## Helper functions for setting up Docker images for viash ######## +# expects: ViashDockerBuild + +# ViashDockerInstallationCheck: check whether Docker is installed correctly +# +# examples: +# ViashDockerInstallationCheck +function ViashDockerInstallationCheck { + ViashDebug "Checking whether Docker is installed" + if [ ! command -v docker &> /dev/null ]; then + ViashCritical "Docker doesn't seem to be installed. See 'https://docs.docker.com/get-docker/' for instructions." + exit 1 + fi + + ViashDebug "Checking whether the Docker daemon is running" + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? + [[ $save =~ e ]] && set -e + if [ $out -ne 0 ]; then + ViashCritical "Docker daemon does not seem to be running. Try one of the following:" + ViashCritical "- Try running 'dockerd' in the command line" + ViashCritical "- See https://docs.docker.com/config/daemon/" + exit 1 + fi +} + +# ViashDockerRemoteTagCheck: check whether a Docker image is available +# on a remote. Assumes `docker login` has been performed, if relevant. +# +# $1 : image identifier with format `[registry/]image[:tag]` +# exit code $? : whether or not the image was found +# examples: +# ViashDockerRemoteTagCheck python:latest +# echo $? # returns '0' +# ViashDockerRemoteTagCheck sdaizudceahifu +# echo $? # returns '1' +function ViashDockerRemoteTagCheck { + docker manifest inspect $1 > /dev/null 2> /dev/null +} + +# ViashDockerLocalTagCheck: check whether a Docker image is available locally +# +# $1 : image identifier with format `[registry/]image[:tag]` +# exit code $? : whether or not the image was found +# examples: +# docker pull python:latest +# ViashDockerLocalTagCheck python:latest +# echo $? # returns '0' +# ViashDockerLocalTagCheck sdaizudceahifu +# echo $? # returns '1' +function ViashDockerLocalTagCheck { + [ -n "$(docker images -q $1)" ] +} + +# ViashDockerPull: pull a Docker image +# +# $1 : image identifier with format `[registry/]image[:tag]` +# exit code $? : whether or not the image was found +# examples: +# ViashDockerPull python:latest +# echo $? # returns '0' +# ViashDockerPull sdaizudceahifu +# echo $? # returns '1' +function ViashDockerPull { + ViashNotice "Checking if Docker image is available at '$1'" + if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then + docker pull $1 && return 0 || return 1 + else + local save=$-; set +e + docker pull $1 2> /dev/null > /dev/null + local out=$? + [[ $save =~ e ]] && set -e + if [ $out -ne 0 ]; then + ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." + fi + return $out + fi +} + +# ViashDockerPush: push a Docker image +# +# $1 : image identifier with format `[registry/]image[:tag]` +# exit code $? : whether or not the image was found +# examples: +# ViashDockerPush python:latest +# echo $? # returns '0' +# ViashDockerPush sdaizudceahifu +# echo $? # returns '1' +function ViashDockerPush { + ViashNotice "Pushing image to '$1'" + local save=$-; set +e + local out + if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then + docker push $1 + out=$? + else + docker push $1 2> /dev/null > /dev/null + out=$? + fi + [[ $save =~ e ]] && set -e + if [ $out -eq 0 ]; then + ViashNotice "Container '$1' push succeeded." + else + ViashError "Container '$1' push errored. You might not be logged in or have the necessary permissions." + fi + return $out +} + +# ViashDockerPullElseBuild: pull a Docker image, else build it +# +# $1 : image identifier with format `[registry/]image[:tag]` +# ViashDockerBuild : a Bash function which builds a docker image, takes image identifier as argument. +# examples: +# ViashDockerPullElseBuild mynewcomponent +function ViashDockerPullElseBuild { + local save=$-; set +e + ViashDockerPull $1 + local out=$? + [[ $save =~ e ]] && set -e + if [ $out -ne 0 ]; then + ViashDockerBuild $@ + fi +} + +# ViashDockerSetup: create a Docker image, according to specified docker setup strategy +# +# $1 : image identifier with format `[registry/]image[:tag]` +# $2 : docker setup strategy, see DockerSetupStrategy.scala +# examples: +# ViashDockerSetup mynewcomponent alwaysbuild +function ViashDockerSetup { + local image_id="$1" + local setup_strategy="$2" + if [ "$setup_strategy" == "alwaysbuild" -o "$setup_strategy" == "build" -o "$setup_strategy" == "b" ]; then + ViashDockerBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "alwayspull" -o "$setup_strategy" == "pull" -o "$setup_strategy" == "p" ]; then + ViashDockerPull $image_id + elif [ "$setup_strategy" == "alwayspullelsebuild" -o "$setup_strategy" == "pullelsebuild" ]; then + ViashDockerPullElseBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "alwayspullelsecachedbuild" -o "$setup_strategy" == "pullelsecachedbuild" ]; then + ViashDockerPullElseBuild $image_id $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "alwayscachedbuild" -o "$setup_strategy" == "cachedbuild" -o "$setup_strategy" == "cb" ]; then + ViashDockerBuild $image_id $(ViashDockerBuildArgs "$engine_id") + elif [[ "$setup_strategy" =~ ^ifneedbe ]]; then + local save=$-; set +e + ViashDockerLocalTagCheck $image_id + local outCheck=$? + [[ $save =~ e ]] && set -e + if [ $outCheck -eq 0 ]; then + ViashInfo "Image $image_id already exists" + elif [ "$setup_strategy" == "ifneedbebuild" ]; then + ViashDockerBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "ifneedbecachedbuild" ]; then + ViashDockerBuild $image_id $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "ifneedbepull" ]; then + ViashDockerPull $image_id + elif [ "$setup_strategy" == "ifneedbepullelsebuild" ]; then + ViashDockerPullElseBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "ifneedbepullelsecachedbuild" ]; then + ViashDockerPullElseBuild $image_id $(ViashDockerBuildArgs "$engine_id") + else + ViashError "Unrecognised Docker strategy: $setup_strategy" + exit 1 + fi + elif [ "$setup_strategy" == "push" -o "$setup_strategy" == "forcepush" -o "$setup_strategy" == "alwayspush" ]; then + ViashDockerPush "$image_id" + elif [ "$setup_strategy" == "pushifnotpresent" -o "$setup_strategy" == "gentlepush" -o "$setup_strategy" == "maybepush" ]; then + local save=$-; set +e + ViashDockerRemoteTagCheck $image_id + local outCheck=$? + [[ $save =~ e ]] && set -e + if [ $outCheck -eq 0 ]; then + ViashNotice "Container '$image_id' exists, doing nothing." + else + ViashNotice "Container '$image_id' does not yet exist." + ViashDockerPush "$image_id" + fi + elif [ "$setup_strategy" == "donothing" -o "$setup_strategy" == "meh" ]; then + ViashNotice "Skipping setup." + else + ViashError "Unrecognised Docker strategy: $setup_strategy" + exit 1 + fi +} + +# ViashDockerCheckCommands: Check whether a docker container has the required commands +# +# $1 : image identifier with format `[registry/]image[:tag]` +# $@ : commands to verify being present +# examples: +# ViashDockerCheckCommands bash:4.0 bash ps foo +function ViashDockerCheckCommands { + local image_id="$1" + shift 1 + local commands="$@" + local save=$-; set +e + local missing # mark 'missing' as local in advance, otherwise the exit code of the command will be missing and always be '0' + missing=$(docker run --rm --entrypoint=sh "$image_id" -c "for command in $commands; do command -v \$command >/dev/null 2>&1; if [ \$? -ne 0 ]; then echo \$command; exit 1; fi; done") + local outCheck=$? + [[ $save =~ e ]] && set -e + if [ $outCheck -ne 0 ]; then + ViashError "Docker container '$image_id' does not contain command '$missing'." + exit 1 + fi +} + +# ViashDockerBuild: build a docker image +# $1 : image identifier with format `[registry/]image[:tag]` +# $... : additional arguments to pass to docker build +# $VIASH_META_TEMP_DIR : temporary directory to store dockerfile & optional resources in +# $VIASH_META_NAME : name of the component +# $VIASH_META_RESOURCES_DIR : directory containing the resources +# $VIASH_VERBOSITY : verbosity level +# exit code $? : whether or not the image was built successfully +function ViashDockerBuild { + local image_id="$1" + shift 1 + + # create temporary directory to store dockerfile & optional resources in + local tmpdir=$(mktemp -d "$VIASH_META_TEMP_DIR/dockerbuild-$VIASH_META_NAME-XXXXXX") + local dockerfile="$tmpdir/Dockerfile" + function clean_up { + rm -rf "$tmpdir" + } + trap clean_up EXIT + + # store dockerfile and resources + ViashDockerfile "$VIASH_ENGINE_ID" > "$dockerfile" + + # generate the build command + local docker_build_cmd="docker build -t '$image_id' $@ '$VIASH_META_RESOURCES_DIR' -f '$dockerfile'" + + # build the container + ViashNotice "Building container '$image_id' with Dockerfile" + ViashInfo "$docker_build_cmd" + local save=$-; set +e + if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then + eval $docker_build_cmd + else + eval $docker_build_cmd &> "$tmpdir/docker_build.log" + fi + + # check exit code + local out=$? + [[ $save =~ e ]] && set -e + if [ $out -ne 0 ]; then + ViashError "Error occurred while building container '$image_id'" + if [ $VIASH_VERBOSITY -lt $VIASH_LOGCODE_INFO ]; then + ViashError "Transcript: --------------------------------" + cat "$tmpdir/docker_build.log" + ViashError "End of transcript --------------------------" + fi + exit 1 + fi +} + +######## End of helper functions for setting up Docker images for viash ######## + +# ViashDockerFile: print the dockerfile to stdout +# $1 : engine identifier +# return : dockerfile required to run this component +# examples: +# ViashDockerFile +function ViashDockerfile { + local engine_id="$1" + + if [[ "$engine_id" == "docker" ]]; then + cat << 'VIASHDOCKER' +FROM debian:stable-slim +ENTRYPOINT [] +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y bedtools procps && \ + rm -rf /var/lib/apt/lists/* + +RUN echo "bedtools: \"$(bedtools --version | sed -n 's/^bedtools //p')\"" > /var/software_versions.txt + +LABEL org.opencontainers.image.authors="Theodoro Gasperin Terra Camargo" +LABEL org.opencontainers.image.description="Companion container for running component bedtools bedtools_merge" +LABEL org.opencontainers.image.created="2024-09-02T13:02:37Z" +LABEL org.opencontainers.image.source="https://github.com/arq5x/bedtools2" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" +LABEL org.opencontainers.image.version="main" + +VIASHDOCKER + fi +} + +# ViashDockerBuildArgs: return the arguments to pass to docker build +# $1 : engine identifier +# return : arguments to pass to docker build +function ViashDockerBuildArgs { + local engine_id="$1" + + if [[ "$engine_id" == "docker" ]]; then + echo "" + fi +} + +# ViashAbsolutePath: generate absolute path from relative path +# borrowed from https://stackoverflow.com/a/21951256 +# $1 : relative filename +# return : absolute path +# examples: +# ViashAbsolutePath some_file.txt # returns /path/to/some_file.txt +# ViashAbsolutePath /foo/bar/.. # returns /foo +function ViashAbsolutePath { + local thePath + local parr + local outp + local len + if [[ ! "$1" =~ ^/ ]]; then + thePath="$PWD/$1" + else + thePath="$1" + fi + echo "$thePath" | ( + IFS=/ + read -a parr + declare -a outp + for i in "${parr[@]}"; do + case "$i" in + ''|.) continue ;; + ..) + len=${#outp[@]} + if ((len==0)); then + continue + else + unset outp[$((len-1))] + fi + ;; + *) + len=${#outp[@]} + outp[$len]="$i" + ;; + esac + done + echo /"${outp[*]}" + ) +} +# ViashDockerAutodetectMount: auto configuring docker mounts from parameters +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts +# examples: +# ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' +# ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' +function ViashDockerAutodetectMount { + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name + if [ -d "$abs_path" ]; then + mount_source="$abs_path" + base_name="" + else + mount_source=`dirname "$abs_path"` + base_name=`basename "$abs_path"` + fi + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" + if [ -z "$base_name" ]; then + echo "$mount_target" + else + echo "$mount_target/$base_name" + fi +} +function ViashDockerAutodetectMountArg { + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name + if [ -d "$abs_path" ]; then + mount_source="$abs_path" + base_name="" + else + mount_source=`dirname "$abs_path"` + base_name=`basename "$abs_path"` + fi + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" + ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" + echo "--volume=\"$mount_source:$mount_target\"" +} +function ViashDockerStripAutomount { + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" +} +# initialise variables +VIASH_DIRECTORY_MOUNTS=() + +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + +# initialise docker variables +VIASH_DOCKER_RUN_ARGS=(-i --rm) + +# initialise array +VIASH_POSITIONAL_ARGS='' + +while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) + ViashHelp + exit + ;; + ---v|---verbose) + let "VIASH_VERBOSITY=VIASH_VERBOSITY+1" + shift 1 + ;; + ---verbosity) + VIASH_VERBOSITY="$2" + shift 2 + ;; + ---verbosity=*) + VIASH_VERBOSITY="$(ViashRemoveFlags "$1")" + shift 1 + ;; + --version) + echo "bedtools_merge main" + exit + ;; + --input) + [ -n "$VIASH_PAR_INPUT" ] && ViashError Bad arguments for option \'--input\': \'$VIASH_PAR_INPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_INPUT="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --input. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --input=*) + [ -n "$VIASH_PAR_INPUT" ] && ViashError Bad arguments for option \'--input=*\': \'$VIASH_PAR_INPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_INPUT=$(ViashRemoveFlags "$1") + shift 1 + ;; + -i) + [ -n "$VIASH_PAR_INPUT" ] && ViashError Bad arguments for option \'-i\': \'$VIASH_PAR_INPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_INPUT="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to -i. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --output) + [ -n "$VIASH_PAR_OUTPUT" ] && ViashError Bad arguments for option \'--output\': \'$VIASH_PAR_OUTPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_OUTPUT="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --output. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --output=*) + [ -n "$VIASH_PAR_OUTPUT" ] && ViashError Bad arguments for option \'--output=*\': \'$VIASH_PAR_OUTPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_OUTPUT=$(ViashRemoveFlags "$1") + shift 1 + ;; + --strand) + [ -n "$VIASH_PAR_STRAND" ] && ViashError Bad arguments for option \'--strand\': \'$VIASH_PAR_STRAND\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_STRAND=true + shift 1 + ;; + -s) + [ -n "$VIASH_PAR_STRAND" ] && ViashError Bad arguments for option \'-s\': \'$VIASH_PAR_STRAND\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_STRAND=true + shift 1 + ;; + --specific_strand) + [ -n "$VIASH_PAR_SPECIFIC_STRAND" ] && ViashError Bad arguments for option \'--specific_strand\': \'$VIASH_PAR_SPECIFIC_STRAND\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_SPECIFIC_STRAND="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --specific_strand. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --specific_strand=*) + [ -n "$VIASH_PAR_SPECIFIC_STRAND" ] && ViashError Bad arguments for option \'--specific_strand=*\': \'$VIASH_PAR_SPECIFIC_STRAND\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_SPECIFIC_STRAND=$(ViashRemoveFlags "$1") + shift 1 + ;; + -S) + [ -n "$VIASH_PAR_SPECIFIC_STRAND" ] && ViashError Bad arguments for option \'-S\': \'$VIASH_PAR_SPECIFIC_STRAND\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_SPECIFIC_STRAND="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to -S. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --distance) + [ -n "$VIASH_PAR_DISTANCE" ] && ViashError Bad arguments for option \'--distance\': \'$VIASH_PAR_DISTANCE\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_DISTANCE="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --distance. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --distance=*) + [ -n "$VIASH_PAR_DISTANCE" ] && ViashError Bad arguments for option \'--distance=*\': \'$VIASH_PAR_DISTANCE\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_DISTANCE=$(ViashRemoveFlags "$1") + shift 1 + ;; + -d) + [ -n "$VIASH_PAR_DISTANCE" ] && ViashError Bad arguments for option \'-d\': \'$VIASH_PAR_DISTANCE\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_DISTANCE="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to -d. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --columns) + [ -n "$VIASH_PAR_COLUMNS" ] && ViashError Bad arguments for option \'--columns\': \'$VIASH_PAR_COLUMNS\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_COLUMNS="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --columns. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --columns=*) + [ -n "$VIASH_PAR_COLUMNS" ] && ViashError Bad arguments for option \'--columns=*\': \'$VIASH_PAR_COLUMNS\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_COLUMNS=$(ViashRemoveFlags "$1") + shift 1 + ;; + -c) + [ -n "$VIASH_PAR_COLUMNS" ] && ViashError Bad arguments for option \'-c\': \'$VIASH_PAR_COLUMNS\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_COLUMNS="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to -c. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --operation) + [ -n "$VIASH_PAR_OPERATION" ] && ViashError Bad arguments for option \'--operation\': \'$VIASH_PAR_OPERATION\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_OPERATION="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --operation. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --operation=*) + [ -n "$VIASH_PAR_OPERATION" ] && ViashError Bad arguments for option \'--operation=*\': \'$VIASH_PAR_OPERATION\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_OPERATION=$(ViashRemoveFlags "$1") + shift 1 + ;; + -o) + [ -n "$VIASH_PAR_OPERATION" ] && ViashError Bad arguments for option \'-o\': \'$VIASH_PAR_OPERATION\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_OPERATION="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to -o. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --delimiter) + [ -n "$VIASH_PAR_DELIMITER" ] && ViashError Bad arguments for option \'--delimiter\': \'$VIASH_PAR_DELIMITER\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_DELIMITER="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --delimiter. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --delimiter=*) + [ -n "$VIASH_PAR_DELIMITER" ] && ViashError Bad arguments for option \'--delimiter=*\': \'$VIASH_PAR_DELIMITER\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_DELIMITER=$(ViashRemoveFlags "$1") + shift 1 + ;; + -delim) + [ -n "$VIASH_PAR_DELIMITER" ] && ViashError Bad arguments for option \'-delim\': \'$VIASH_PAR_DELIMITER\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_DELIMITER="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to -delim. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --precision) + [ -n "$VIASH_PAR_PRECISION" ] && ViashError Bad arguments for option \'--precision\': \'$VIASH_PAR_PRECISION\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_PRECISION="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --precision. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --precision=*) + [ -n "$VIASH_PAR_PRECISION" ] && ViashError Bad arguments for option \'--precision=*\': \'$VIASH_PAR_PRECISION\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_PRECISION=$(ViashRemoveFlags "$1") + shift 1 + ;; + -prec) + [ -n "$VIASH_PAR_PRECISION" ] && ViashError Bad arguments for option \'-prec\': \'$VIASH_PAR_PRECISION\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_PRECISION="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to -prec. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --bed) + [ -n "$VIASH_PAR_BED" ] && ViashError Bad arguments for option \'--bed\': \'$VIASH_PAR_BED\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_BED=true + shift 1 + ;; + --header) + [ -n "$VIASH_PAR_HEADER" ] && ViashError Bad arguments for option \'--header\': \'$VIASH_PAR_HEADER\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_HEADER=true + shift 1 + ;; + --no_buffer) + [ -n "$VIASH_PAR_NO_BUFFER" ] && ViashError Bad arguments for option \'--no_buffer\': \'$VIASH_PAR_NO_BUFFER\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_NO_BUFFER=true + shift 1 + ;; + -nobuf) + [ -n "$VIASH_PAR_NO_BUFFER" ] && ViashError Bad arguments for option \'-nobuf\': \'$VIASH_PAR_NO_BUFFER\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_NO_BUFFER=true + shift 1 + ;; + ---engine) + VIASH_ENGINE_ID="$2" + shift 2 + ;; + ---engine=*) + VIASH_ENGINE_ID="$(ViashRemoveFlags "$1")" + shift 1 + ;; + ---setup) + VIASH_MODE='setup' + VIASH_SETUP_STRATEGY="$2" + shift 2 + ;; + ---setup=*) + VIASH_MODE='setup' + VIASH_SETUP_STRATEGY="$(ViashRemoveFlags "$1")" + shift 1 + ;; + ---dockerfile) + VIASH_MODE='dockerfile' + shift 1 + ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; + ---debug) + VIASH_MODE='debug' + shift 1 + ;; + ---cpus) + [ -n "$VIASH_META_CPUS" ] && ViashError Bad arguments for option \'---cpus\': \'$VIASH_META_CPUS\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_META_CPUS="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to ---cpus. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + ---cpus=*) + [ -n "$VIASH_META_CPUS" ] && ViashError Bad arguments for option \'---cpus=*\': \'$VIASH_META_CPUS\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_META_CPUS=$(ViashRemoveFlags "$1") + shift 1 + ;; + ---memory) + [ -n "$VIASH_META_MEMORY" ] && ViashError Bad arguments for option \'---memory\': \'$VIASH_META_MEMORY\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_META_MEMORY="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to ---memory. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + ---memory=*) + [ -n "$VIASH_META_MEMORY" ] && ViashError Bad arguments for option \'---memory=*\': \'$VIASH_META_MEMORY\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_META_MEMORY=$(ViashRemoveFlags "$1") + shift 1 + ;; + *) # positional arg or unknown option + # since the positional args will be eval'd, can we always quote, instead of using ViashQuote + VIASH_POSITIONAL_ARGS="$VIASH_POSITIONAL_ARGS '$1'" + [[ $1 == -* ]] && ViashWarning $1 looks like a parameter but is not a defined parameter and will instead be treated as a positional argument. Use "--help" to get more information on the parameters. + shift # past argument + ;; + esac +done + +# parse positional parameters +eval set -- $VIASH_POSITIONAL_ARGS + + +if [ "$VIASH_ENGINE_ID" == "native" ] ; then + VIASH_ENGINE_TYPE='native' +elif [ "$VIASH_ENGINE_ID" == "docker" ] ; then + VIASH_ENGINE_TYPE='docker' +else + ViashError "Engine '$VIASH_ENGINE_ID' is not recognized. Options are: docker, native." + exit 1 +fi + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + # check if docker is installed properly + ViashDockerInstallationCheck + + # determine docker image id + if [[ "$VIASH_ENGINE_ID" == 'docker' ]]; then + VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/biobox/bedtools/bedtools_merge:main' + fi + + # print dockerfile + if [ "$VIASH_MODE" == "dockerfile" ]; then + ViashDockerfile "$VIASH_ENGINE_ID" + exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 + + # enter docker container + elif [[ "$VIASH_MODE" == "debug" ]]; then + VIASH_CMD="docker run --entrypoint=bash ${VIASH_DOCKER_RUN_ARGS[@]} -v '$(pwd)':/pwd --workdir /pwd -t $VIASH_DOCKER_IMAGE_ID" + ViashNotice "+ $VIASH_CMD" + eval $VIASH_CMD + exit + + # build docker image + elif [ "$VIASH_MODE" == "setup" ]; then + ViashDockerSetup "$VIASH_DOCKER_IMAGE_ID" "$VIASH_SETUP_STRATEGY" + ViashDockerCheckCommands "$VIASH_DOCKER_IMAGE_ID" 'ps' 'bash' + exit 0 + fi + + # check if docker image exists + ViashDockerSetup "$VIASH_DOCKER_IMAGE_ID" ifneedbepullelsecachedbuild + ViashDockerCheckCommands "$VIASH_DOCKER_IMAGE_ID" 'ps' 'bash' +fi + +# setting computational defaults + +# helper function for parsing memory strings +function ViashMemoryAsBytes { + local memory=`echo "$1" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]'` + local memory_regex='^([0-9]+)([kmgtp]i?b?|b)$' + if [[ $memory =~ $memory_regex ]]; then + local number=${memory/[^0-9]*/} + local symbol=${memory/*[0-9]/} + + case $symbol in + b) memory_b=$number ;; + kb|k) memory_b=$(( $number * 1000 )) ;; + mb|m) memory_b=$(( $number * 1000 * 1000 )) ;; + gb|g) memory_b=$(( $number * 1000 * 1000 * 1000 )) ;; + tb|t) memory_b=$(( $number * 1000 * 1000 * 1000 * 1000 )) ;; + pb|p) memory_b=$(( $number * 1000 * 1000 * 1000 * 1000 * 1000 )) ;; + kib|ki) memory_b=$(( $number * 1024 )) ;; + mib|mi) memory_b=$(( $number * 1024 * 1024 )) ;; + gib|gi) memory_b=$(( $number * 1024 * 1024 * 1024 )) ;; + tib|ti) memory_b=$(( $number * 1024 * 1024 * 1024 * 1024 )) ;; + pib|pi) memory_b=$(( $number * 1024 * 1024 * 1024 * 1024 * 1024 )) ;; + esac + echo "$memory_b" + fi +} +# compute memory in different units +if [ ! -z ${VIASH_META_MEMORY+x} ]; then + VIASH_META_MEMORY_B=`ViashMemoryAsBytes $VIASH_META_MEMORY` + # do not define other variables if memory_b is an empty string + if [ ! -z "$VIASH_META_MEMORY_B" ]; then + VIASH_META_MEMORY_KB=$(( ($VIASH_META_MEMORY_B+999) / 1000 )) + VIASH_META_MEMORY_MB=$(( ($VIASH_META_MEMORY_KB+999) / 1000 )) + VIASH_META_MEMORY_GB=$(( ($VIASH_META_MEMORY_MB+999) / 1000 )) + VIASH_META_MEMORY_TB=$(( ($VIASH_META_MEMORY_GB+999) / 1000 )) + VIASH_META_MEMORY_PB=$(( ($VIASH_META_MEMORY_TB+999) / 1000 )) + VIASH_META_MEMORY_KIB=$(( ($VIASH_META_MEMORY_B+1023) / 1024 )) + VIASH_META_MEMORY_MIB=$(( ($VIASH_META_MEMORY_KIB+1023) / 1024 )) + VIASH_META_MEMORY_GIB=$(( ($VIASH_META_MEMORY_MIB+1023) / 1024 )) + VIASH_META_MEMORY_TIB=$(( ($VIASH_META_MEMORY_GIB+1023) / 1024 )) + VIASH_META_MEMORY_PIB=$(( ($VIASH_META_MEMORY_TIB+1023) / 1024 )) + else + # unset memory if string is empty + unset $VIASH_META_MEMORY_B + fi +fi +# unset nproc if string is empty +if [ -z "$VIASH_META_CPUS" ]; then + unset $VIASH_META_CPUS +fi + + +# check whether required parameters exist +if [ -z ${VIASH_PAR_INPUT+x} ]; then + ViashError '--input' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_PAR_OUTPUT+x} ]; then + ViashError '--output' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_NAME+x} ]; then + ViashError 'name' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_FUNCTIONALITY_NAME+x} ]; then + ViashError 'functionality_name' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_RESOURCES_DIR+x} ]; then + ViashError 'resources_dir' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_EXECUTABLE+x} ]; then + ViashError 'executable' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_CONFIG+x} ]; then + ViashError 'config' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_TEMP_DIR+x} ]; then + ViashError 'temp_dir' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi + +# filling in defaults +if [ -z ${VIASH_PAR_STRAND+x} ]; then + VIASH_PAR_STRAND="false" +fi +if [ -z ${VIASH_PAR_DELIMITER+x} ]; then + VIASH_PAR_DELIMITER="," +fi +if [ -z ${VIASH_PAR_BED+x} ]; then + VIASH_PAR_BED="false" +fi +if [ -z ${VIASH_PAR_HEADER+x} ]; then + VIASH_PAR_HEADER="false" +fi +if [ -z ${VIASH_PAR_NO_BUFFER+x} ]; then + VIASH_PAR_NO_BUFFER="false" +fi + +# check whether required files exist +if [ ! -z "$VIASH_PAR_INPUT" ] && [ ! -e "$VIASH_PAR_INPUT" ]; then + ViashError "Input file '$VIASH_PAR_INPUT' does not exist." + exit 1 +fi + +# check whether parameters values are of the right type +if [[ -n "$VIASH_PAR_STRAND" ]]; then + if ! [[ "$VIASH_PAR_STRAND" =~ ^(true|True|TRUE|false|False|FALSE|yes|Yes|YES|no|No|NO)$ ]]; then + ViashError '--strand' has to be a boolean_true. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_PAR_DISTANCE" ]]; then + if ! [[ "$VIASH_PAR_DISTANCE" =~ ^[-+]?[0-9]+$ ]]; then + ViashError '--distance' has to be an integer. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_PAR_COLUMNS" ]]; then + if ! [[ "$VIASH_PAR_COLUMNS" =~ ^[-+]?[0-9]+$ ]]; then + ViashError '--columns' has to be an integer. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_PAR_PRECISION" ]]; then + if ! [[ "$VIASH_PAR_PRECISION" =~ ^[-+]?[0-9]+$ ]]; then + ViashError '--precision' has to be an integer. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_PAR_BED" ]]; then + if ! [[ "$VIASH_PAR_BED" =~ ^(true|True|TRUE|false|False|FALSE|yes|Yes|YES|no|No|NO)$ ]]; then + ViashError '--bed' has to be a boolean_true. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_PAR_HEADER" ]]; then + if ! [[ "$VIASH_PAR_HEADER" =~ ^(true|True|TRUE|false|False|FALSE|yes|Yes|YES|no|No|NO)$ ]]; then + ViashError '--header' has to be a boolean_true. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_PAR_NO_BUFFER" ]]; then + if ! [[ "$VIASH_PAR_NO_BUFFER" =~ ^(true|True|TRUE|false|False|FALSE|yes|Yes|YES|no|No|NO)$ ]]; then + ViashError '--no_buffer' has to be a boolean_true. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_CPUS" ]]; then + if ! [[ "$VIASH_META_CPUS" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'cpus' has to be an integer. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_B" ]]; then + if ! [[ "$VIASH_META_MEMORY_B" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_b' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_KB" ]]; then + if ! [[ "$VIASH_META_MEMORY_KB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_kb' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_MB" ]]; then + if ! [[ "$VIASH_META_MEMORY_MB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_mb' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_GB" ]]; then + if ! [[ "$VIASH_META_MEMORY_GB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_gb' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_TB" ]]; then + if ! [[ "$VIASH_META_MEMORY_TB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_tb' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_PB" ]]; then + if ! [[ "$VIASH_META_MEMORY_PB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_pb' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_KIB" ]]; then + if ! [[ "$VIASH_META_MEMORY_KIB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_kib' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_MIB" ]]; then + if ! [[ "$VIASH_META_MEMORY_MIB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_mib' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_GIB" ]]; then + if ! [[ "$VIASH_META_MEMORY_GIB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_gib' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_TIB" ]]; then + if ! [[ "$VIASH_META_MEMORY_TIB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_tib' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_PIB" ]]; then + if ! [[ "$VIASH_META_MEMORY_PIB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_pib' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi + +# check whether value is belongs to a set of choices +if [ ! -z "$VIASH_PAR_SPECIFIC_STRAND" ]; then + VIASH_PAR_SPECIFIC_STRAND_CHOICES=("+;-") + IFS=';' + set -f + if ! [[ ";${VIASH_PAR_SPECIFIC_STRAND_CHOICES[*]};" =~ ";$VIASH_PAR_SPECIFIC_STRAND;" ]]; then + ViashError '--specific_strand' specified value of \'$VIASH_PAR_SPECIFIC_STRAND\' is not in the list of allowed values. Use "--help" to get more information on the parameters. + exit 1 + fi + set +f + unset IFS +fi + +# create parent directories of output files, if so desired +if [ ! -z "$VIASH_PAR_OUTPUT" ] && [ ! -d "$(dirname "$VIASH_PAR_OUTPUT")" ]; then + mkdir -p "$(dirname "$VIASH_PAR_OUTPUT")" +fi + +if [ "$VIASH_ENGINE_ID" == "native" ] ; then + if [ "$VIASH_MODE" == "run" ]; then + VIASH_CMD="bash" + else + ViashError "Engine '$VIASH_ENGINE_ID' does not support mode '$VIASH_MODE'." + exit 1 + fi +fi + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + # detect volumes from file arguments + VIASH_CHOWN_VARS=() +if [ ! -z "$VIASH_PAR_INPUT" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_PAR_INPUT")" ) + VIASH_PAR_INPUT=$(ViashDockerAutodetectMount "$VIASH_PAR_INPUT") +fi +if [ ! -z "$VIASH_PAR_OUTPUT" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_PAR_OUTPUT")" ) + VIASH_PAR_OUTPUT=$(ViashDockerAutodetectMount "$VIASH_PAR_OUTPUT") + VIASH_CHOWN_VARS+=( "$VIASH_PAR_OUTPUT" ) +fi +if [ ! -z "$VIASH_META_RESOURCES_DIR" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_RESOURCES_DIR")" ) + VIASH_META_RESOURCES_DIR=$(ViashDockerAutodetectMount "$VIASH_META_RESOURCES_DIR") +fi +if [ ! -z "$VIASH_META_EXECUTABLE" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_EXECUTABLE")" ) + VIASH_META_EXECUTABLE=$(ViashDockerAutodetectMount "$VIASH_META_EXECUTABLE") +fi +if [ ! -z "$VIASH_META_CONFIG" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_CONFIG")" ) + VIASH_META_CONFIG=$(ViashDockerAutodetectMount "$VIASH_META_CONFIG") +fi +if [ ! -z "$VIASH_META_TEMP_DIR" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_TEMP_DIR")" ) + VIASH_META_TEMP_DIR=$(ViashDockerAutodetectMount "$VIASH_META_TEMP_DIR") +fi + + # get unique mounts + VIASH_UNIQUE_MOUNTS=($(for val in "${VIASH_DIRECTORY_MOUNTS[@]}"; do echo "$val"; done | sort -u)) +fi + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + # change file ownership + function ViashPerformChown { + if (( ${#VIASH_CHOWN_VARS[@]} )); then + set +e + VIASH_CMD="docker run --entrypoint=bash --rm ${VIASH_UNIQUE_MOUNTS[@]} $VIASH_DOCKER_IMAGE_ID -c 'chown $(id -u):$(id -g) --silent --recursive ${VIASH_CHOWN_VARS[@]}'" + ViashDebug "+ $VIASH_CMD" + eval $VIASH_CMD + set -e + fi + } + trap ViashPerformChown EXIT +fi + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + # helper function for filling in extra docker args + if [ ! -z "$VIASH_META_MEMORY_B" ]; then + VIASH_DOCKER_RUN_ARGS+=("--memory=${VIASH_META_MEMORY_B}") + fi + if [ ! -z "$VIASH_META_CPUS" ]; then + VIASH_DOCKER_RUN_ARGS+=("--cpus=${VIASH_META_CPUS}") + fi +fi + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + VIASH_CMD="docker run --entrypoint=bash ${VIASH_DOCKER_RUN_ARGS[@]} ${VIASH_UNIQUE_MOUNTS[@]} $VIASH_DOCKER_IMAGE_ID" +fi + + +# set dependency paths + + +ViashDebug "Running command: $(echo $VIASH_CMD)" +cat << VIASHEOF | eval $VIASH_CMD +set -e +tempscript=\$(mktemp "$VIASH_META_TEMP_DIR/viash-run-bedtools_merge-XXXXXX").sh +function clean_up { + rm "\$tempscript" +} +function interrupt { + echo -e "\nCTRL-C Pressed..." + exit 1 +} +trap clean_up EXIT +trap interrupt INT SIGINT +cat > "\$tempscript" << 'VIASHMAIN' +#!/bin/bash + +## VIASH START +# The following code has been auto-generated by Viash. +$( if [ ! -z ${VIASH_PAR_INPUT+x} ]; then echo "${VIASH_PAR_INPUT}" | sed "s#'#'\"'\"'#g;s#.*#par_input='&'#" ; else echo "# par_input="; fi ) +$( if [ ! -z ${VIASH_PAR_OUTPUT+x} ]; then echo "${VIASH_PAR_OUTPUT}" | sed "s#'#'\"'\"'#g;s#.*#par_output='&'#" ; else echo "# par_output="; fi ) +$( if [ ! -z ${VIASH_PAR_STRAND+x} ]; then echo "${VIASH_PAR_STRAND}" | sed "s#'#'\"'\"'#g;s#.*#par_strand='&'#" ; else echo "# par_strand="; fi ) +$( if [ ! -z ${VIASH_PAR_SPECIFIC_STRAND+x} ]; then echo "${VIASH_PAR_SPECIFIC_STRAND}" | sed "s#'#'\"'\"'#g;s#.*#par_specific_strand='&'#" ; else echo "# par_specific_strand="; fi ) +$( if [ ! -z ${VIASH_PAR_DISTANCE+x} ]; then echo "${VIASH_PAR_DISTANCE}" | sed "s#'#'\"'\"'#g;s#.*#par_distance='&'#" ; else echo "# par_distance="; fi ) +$( if [ ! -z ${VIASH_PAR_COLUMNS+x} ]; then echo "${VIASH_PAR_COLUMNS}" | sed "s#'#'\"'\"'#g;s#.*#par_columns='&'#" ; else echo "# par_columns="; fi ) +$( if [ ! -z ${VIASH_PAR_OPERATION+x} ]; then echo "${VIASH_PAR_OPERATION}" | sed "s#'#'\"'\"'#g;s#.*#par_operation='&'#" ; else echo "# par_operation="; fi ) +$( if [ ! -z ${VIASH_PAR_DELIMITER+x} ]; then echo "${VIASH_PAR_DELIMITER}" | sed "s#'#'\"'\"'#g;s#.*#par_delimiter='&'#" ; else echo "# par_delimiter="; fi ) +$( if [ ! -z ${VIASH_PAR_PRECISION+x} ]; then echo "${VIASH_PAR_PRECISION}" | sed "s#'#'\"'\"'#g;s#.*#par_precision='&'#" ; else echo "# par_precision="; fi ) +$( if [ ! -z ${VIASH_PAR_BED+x} ]; then echo "${VIASH_PAR_BED}" | sed "s#'#'\"'\"'#g;s#.*#par_bed='&'#" ; else echo "# par_bed="; fi ) +$( if [ ! -z ${VIASH_PAR_HEADER+x} ]; then echo "${VIASH_PAR_HEADER}" | sed "s#'#'\"'\"'#g;s#.*#par_header='&'#" ; else echo "# par_header="; fi ) +$( if [ ! -z ${VIASH_PAR_NO_BUFFER+x} ]; then echo "${VIASH_PAR_NO_BUFFER}" | sed "s#'#'\"'\"'#g;s#.*#par_no_buffer='&'#" ; else echo "# par_no_buffer="; fi ) +$( if [ ! -z ${VIASH_META_NAME+x} ]; then echo "${VIASH_META_NAME}" | sed "s#'#'\"'\"'#g;s#.*#meta_name='&'#" ; else echo "# meta_name="; fi ) +$( if [ ! -z ${VIASH_META_FUNCTIONALITY_NAME+x} ]; then echo "${VIASH_META_FUNCTIONALITY_NAME}" | sed "s#'#'\"'\"'#g;s#.*#meta_functionality_name='&'#" ; else echo "# meta_functionality_name="; fi ) +$( if [ ! -z ${VIASH_META_RESOURCES_DIR+x} ]; then echo "${VIASH_META_RESOURCES_DIR}" | sed "s#'#'\"'\"'#g;s#.*#meta_resources_dir='&'#" ; else echo "# meta_resources_dir="; fi ) +$( if [ ! -z ${VIASH_META_EXECUTABLE+x} ]; then echo "${VIASH_META_EXECUTABLE}" | sed "s#'#'\"'\"'#g;s#.*#meta_executable='&'#" ; else echo "# meta_executable="; fi ) +$( if [ ! -z ${VIASH_META_CONFIG+x} ]; then echo "${VIASH_META_CONFIG}" | sed "s#'#'\"'\"'#g;s#.*#meta_config='&'#" ; else echo "# meta_config="; fi ) +$( if [ ! -z ${VIASH_META_TEMP_DIR+x} ]; then echo "${VIASH_META_TEMP_DIR}" | sed "s#'#'\"'\"'#g;s#.*#meta_temp_dir='&'#" ; else echo "# meta_temp_dir="; fi ) +$( if [ ! -z ${VIASH_META_CPUS+x} ]; then echo "${VIASH_META_CPUS}" | sed "s#'#'\"'\"'#g;s#.*#meta_cpus='&'#" ; else echo "# meta_cpus="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_B+x} ]; then echo "${VIASH_META_MEMORY_B}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_b='&'#" ; else echo "# meta_memory_b="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_KB+x} ]; then echo "${VIASH_META_MEMORY_KB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_kb='&'#" ; else echo "# meta_memory_kb="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_MB+x} ]; then echo "${VIASH_META_MEMORY_MB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_mb='&'#" ; else echo "# meta_memory_mb="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_GB+x} ]; then echo "${VIASH_META_MEMORY_GB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_gb='&'#" ; else echo "# meta_memory_gb="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_TB+x} ]; then echo "${VIASH_META_MEMORY_TB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_tb='&'#" ; else echo "# meta_memory_tb="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_PB+x} ]; then echo "${VIASH_META_MEMORY_PB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_pb='&'#" ; else echo "# meta_memory_pb="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_KIB+x} ]; then echo "${VIASH_META_MEMORY_KIB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_kib='&'#" ; else echo "# meta_memory_kib="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_MIB+x} ]; then echo "${VIASH_META_MEMORY_MIB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_mib='&'#" ; else echo "# meta_memory_mib="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_GIB+x} ]; then echo "${VIASH_META_MEMORY_GIB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_gib='&'#" ; else echo "# meta_memory_gib="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_TIB+x} ]; then echo "${VIASH_META_MEMORY_TIB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_tib='&'#" ; else echo "# meta_memory_tib="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_PIB+x} ]; then echo "${VIASH_META_MEMORY_PIB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_pib='&'#" ; else echo "# meta_memory_pib="; fi ) + +## VIASH END + +# Exit on error +set -eo pipefail + +# Unset parameters +unset_if_false=( + par_strand + par_bed + par_header + par_no_buffer +) + +for par in \${unset_if_false[@]}; do + test_val="\${!par}" + [[ "\$test_val" == "false" ]] && unset \$par +done + +# Execute bedtools merge with the provided arguments +bedtools merge \\ + \${par_strand:+-s} \\ + \${par_specific_strand:+-S "\$par_specific_strand"} \\ + \${par_bed:+-bed} \\ + \${par_header:+-header} \\ + \${par_no_buffer:+-nobuf} \\ + \${par_distance:+-d "\$par_distance"} \\ + \${par_columns:+-c "\$par_columns"} \\ + \${par_operation:+-o "\$par_operation"} \\ + \${par_delimiter:+-delim "\$par_delimiter"} \\ + \${par_precision:+-prec "\$par_precision"} \\ + -i "\$par_input" \\ + > "\$par_output" +VIASHMAIN +bash "\$tempscript" & +wait "\$!" + +VIASHEOF + + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + # strip viash automount from file paths + + if [ ! -z "$VIASH_PAR_INPUT" ]; then + VIASH_PAR_INPUT=$(ViashDockerStripAutomount "$VIASH_PAR_INPUT") + fi + if [ ! -z "$VIASH_PAR_OUTPUT" ]; then + VIASH_PAR_OUTPUT=$(ViashDockerStripAutomount "$VIASH_PAR_OUTPUT") + fi + if [ ! -z "$VIASH_META_RESOURCES_DIR" ]; then + VIASH_META_RESOURCES_DIR=$(ViashDockerStripAutomount "$VIASH_META_RESOURCES_DIR") + fi + if [ ! -z "$VIASH_META_EXECUTABLE" ]; then + VIASH_META_EXECUTABLE=$(ViashDockerStripAutomount "$VIASH_META_EXECUTABLE") + fi + if [ ! -z "$VIASH_META_CONFIG" ]; then + VIASH_META_CONFIG=$(ViashDockerStripAutomount "$VIASH_META_CONFIG") + fi + if [ ! -z "$VIASH_META_TEMP_DIR" ]; then + VIASH_META_TEMP_DIR=$(ViashDockerStripAutomount "$VIASH_META_TEMP_DIR") + fi +fi + + +# check whether required files exist +if [ ! -z "$VIASH_PAR_OUTPUT" ] && [ ! -e "$VIASH_PAR_OUTPUT" ]; then + ViashError "Output file '$VIASH_PAR_OUTPUT' does not exist." + exit 1 +fi + + +exit 0 diff --git a/target/executable/bedtools/bedtools_sort/.config.vsh.yaml b/target/executable/bedtools/bedtools_sort/.config.vsh.yaml index 0fbd5d4b..0b848068 100644 --- a/target/executable/bedtools/bedtools_sort/.config.vsh.yaml +++ b/target/executable/bedtools/bedtools_sort/.config.vsh.yaml @@ -221,15 +221,15 @@ build_info: engine: "docker|native" output: "target/executable/bedtools/bedtools_sort" executable: "target/executable/bedtools/bedtools_sort/bedtools_sort" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/bedtools/bedtools_sort/bedtools_sort b/target/executable/bedtools/bedtools_sort/bedtools_sort index 9c11f8ac..d9fef0e8 100755 --- a/target/executable/bedtools/bedtools_sort/bedtools_sort +++ b/target/executable/bedtools/bedtools_sort/bedtools_sort @@ -2,7 +2,7 @@ # bedtools_sort main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -57,24 +57,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -245,9 +245,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -299,9 +299,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -321,7 +321,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -345,9 +346,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -508,9 +509,9 @@ RUN echo "bedtools: \"$(bedtools --version | sed -n 's/^bedtools //p')\"" > /var LABEL org.opencontainers.image.authors="Theodoro Gasperin Terra Camargo" LABEL org.opencontainers.image.description="Companion container for running component bedtools bedtools_sort" -LABEL org.opencontainers.image.created="2024-08-21T11:35:51Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:39Z" LABEL org.opencontainers.image.source="https://github.com/arq5x/bedtools2" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -537,6 +538,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -567,14 +571,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -582,7 +589,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -590,7 +597,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -598,17 +607,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -756,6 +770,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -817,6 +843,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/busco/busco_download_datasets/.config.vsh.yaml b/target/executable/busco/busco_download_datasets/.config.vsh.yaml index cc54dddc..9dcdf852 100644 --- a/target/executable/busco/busco_download_datasets/.config.vsh.yaml +++ b/target/executable/busco/busco_download_datasets/.config.vsh.yaml @@ -157,15 +157,15 @@ build_info: engine: "docker|native" output: "target/executable/busco/busco_download_datasets" executable: "target/executable/busco/busco_download_datasets/busco_download_datasets" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/busco/busco_download_datasets/busco_download_datasets b/target/executable/busco/busco_download_datasets/busco_download_datasets index 5cb56ccf..d5d552ec 100755 --- a/target/executable/busco/busco_download_datasets/busco_download_datasets +++ b/target/executable/busco/busco_download_datasets/busco_download_datasets @@ -2,7 +2,7 @@ # busco_download_datasets main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -57,24 +57,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -215,9 +215,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -269,9 +269,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -291,7 +291,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -315,9 +316,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -474,9 +475,9 @@ RUN busco --version | sed 's/BUSCO\s\(.*\)/busco: "\1"/' > /var/software_version LABEL org.opencontainers.image.authors="Dorien Roosen" LABEL org.opencontainers.image.description="Companion container for running component busco busco_download_datasets" -LABEL org.opencontainers.image.created="2024-08-21T11:35:59Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:47Z" LABEL org.opencontainers.image.source="https://gitlab.com/ezlab/busco" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -503,6 +504,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -533,14 +537,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -548,7 +555,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -556,7 +563,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -564,17 +573,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -647,6 +661,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -708,6 +734,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/busco/busco_list_datasets/.config.vsh.yaml b/target/executable/busco/busco_list_datasets/.config.vsh.yaml index 97a7be14..d7fc735c 100644 --- a/target/executable/busco/busco_list_datasets/.config.vsh.yaml +++ b/target/executable/busco/busco_list_datasets/.config.vsh.yaml @@ -144,15 +144,15 @@ build_info: engine: "docker|native" output: "target/executable/busco/busco_list_datasets" executable: "target/executable/busco/busco_list_datasets/busco_list_datasets" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/busco/busco_list_datasets/busco_list_datasets b/target/executable/busco/busco_list_datasets/busco_list_datasets index 4cecfc37..601b7bb0 100755 --- a/target/executable/busco/busco_list_datasets/busco_list_datasets +++ b/target/executable/busco/busco_list_datasets/busco_list_datasets @@ -2,7 +2,7 @@ # busco_list_datasets main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -57,24 +57,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -205,9 +205,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -259,9 +259,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -281,7 +281,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -305,9 +306,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -464,9 +465,9 @@ RUN busco --version | sed 's/BUSCO\s\(.*\)/busco: "\1"/' > /var/software_version LABEL org.opencontainers.image.authors="Dorien Roosen" LABEL org.opencontainers.image.description="Companion container for running component busco busco_list_datasets" -LABEL org.opencontainers.image.created="2024-08-21T11:35:59Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:47Z" LABEL org.opencontainers.image.source="https://gitlab.com/ezlab/busco" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -493,6 +494,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -523,14 +527,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -538,7 +545,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -546,7 +553,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -554,17 +563,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -632,6 +646,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -693,6 +719,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/busco/busco_run/.config.vsh.yaml b/target/executable/busco/busco_run/.config.vsh.yaml index 54d7680e..454dde97 100644 --- a/target/executable/busco/busco_run/.config.vsh.yaml +++ b/target/executable/busco/busco_run/.config.vsh.yaml @@ -422,15 +422,15 @@ build_info: engine: "docker|native" output: "target/executable/busco/busco_run" executable: "target/executable/busco/busco_run/busco_run" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/busco/busco_run/busco_run b/target/executable/busco/busco_run/busco_run index 73649abc..9c46a6b2 100755 --- a/target/executable/busco/busco_run/busco_run +++ b/target/executable/busco/busco_run/busco_run @@ -2,7 +2,7 @@ # busco_run main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -57,24 +57,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -372,9 +372,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -426,9 +426,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -448,7 +448,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -472,9 +473,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -631,9 +632,9 @@ RUN busco --version | sed 's/BUSCO\s\(.*\)/busco: "\1"/' > /var/software_version LABEL org.opencontainers.image.authors="Dorien Roosen" LABEL org.opencontainers.image.description="Companion container for running component busco busco_run" -LABEL org.opencontainers.image.created="2024-08-21T11:35:59Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:48Z" LABEL org.opencontainers.image.source="https://gitlab.com/ezlab/busco" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -660,6 +661,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -690,14 +694,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -705,7 +712,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -713,7 +720,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -721,17 +730,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -1046,6 +1060,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -1107,6 +1133,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/cutadapt/.config.vsh.yaml b/target/executable/cutadapt/.config.vsh.yaml index 0067ea27..961aa1ac 100644 --- a/target/executable/cutadapt/.config.vsh.yaml +++ b/target/executable/cutadapt/.config.vsh.yaml @@ -739,15 +739,15 @@ build_info: engine: "docker|native" output: "target/executable/cutadapt" executable: "target/executable/cutadapt/cutadapt" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/cutadapt/cutadapt b/target/executable/cutadapt/cutadapt index 98f91342..4c461a3d 100755 --- a/target/executable/cutadapt/cutadapt +++ b/target/executable/cutadapt/cutadapt @@ -2,7 +2,7 @@ # cutadapt main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -57,24 +57,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -568,9 +568,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -622,9 +622,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -644,7 +644,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -668,9 +669,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -830,9 +831,9 @@ RUN cutadapt --version | sed 's/\(.*\)/cutadapt: "\1"/' > /var/software_versions LABEL org.opencontainers.image.authors="Toni Verbeiren" LABEL org.opencontainers.image.description="Companion container for running component cutadapt" -LABEL org.opencontainers.image.created="2024-08-21T11:36:03Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:52Z" LABEL org.opencontainers.image.source="https://github.com/marcelm/cutadapt" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -859,6 +860,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -889,14 +893,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -904,7 +911,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -912,7 +919,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -920,17 +929,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -1705,6 +1719,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -1766,6 +1792,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/falco/.config.vsh.yaml b/target/executable/falco/.config.vsh.yaml index 3b47c8c2..8de36abb 100644 --- a/target/executable/falco/.config.vsh.yaml +++ b/target/executable/falco/.config.vsh.yaml @@ -316,15 +316,15 @@ build_info: engine: "docker|native" output: "target/executable/falco" executable: "target/executable/falco/falco" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/falco/falco b/target/executable/falco/falco index 803a8347..0883a63b 100755 --- a/target/executable/falco/falco +++ b/target/executable/falco/falco @@ -2,7 +2,7 @@ # falco main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -57,24 +57,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -317,9 +317,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -371,9 +371,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -393,7 +393,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -417,9 +418,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -588,9 +589,9 @@ RUN echo "falco: \"$(falco -v | sed -n 's/^falco //p')\"" > /var/software_versio LABEL org.opencontainers.image.authors="Toni Verbeiren" LABEL org.opencontainers.image.description="Companion container for running component falco" -LABEL org.opencontainers.image.created="2024-08-21T11:36:04Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:53Z" LABEL org.opencontainers.image.source="https://github.com/smithlabcode/falco" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -617,6 +618,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -647,14 +651,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -662,7 +669,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -670,7 +677,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -678,17 +687,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -916,6 +930,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -977,6 +1003,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/fastp/.config.vsh.yaml b/target/executable/fastp/.config.vsh.yaml index ea112e85..54a8ef37 100644 --- a/target/executable/fastp/.config.vsh.yaml +++ b/target/executable/fastp/.config.vsh.yaml @@ -1082,15 +1082,15 @@ build_info: engine: "docker|native" output: "target/executable/fastp" executable: "target/executable/fastp/fastp" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/fastp/fastp b/target/executable/fastp/fastp index 3a2124be..a5dd39e8 100755 --- a/target/executable/fastp/fastp +++ b/target/executable/fastp/fastp @@ -2,7 +2,7 @@ # fastp main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -57,24 +57,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -768,9 +768,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -822,9 +822,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -844,7 +844,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -868,9 +869,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -1027,9 +1028,9 @@ RUN fastp --version 2>&1 | sed 's# #: "#;s#$#"#' > /var/software_versions.txt LABEL org.opencontainers.image.authors="Robrecht Cannoodt" LABEL org.opencontainers.image.description="Companion container for running component fastp" -LABEL org.opencontainers.image.created="2024-08-21T11:36:01Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:50Z" LABEL org.opencontainers.image.source="https://github.com/OpenGene/fastp" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -1056,6 +1057,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -1086,14 +1090,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -1101,7 +1108,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -1109,7 +1116,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -1117,17 +1126,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -2103,6 +2117,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -2164,6 +2190,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/featurecounts/.config.vsh.yaml b/target/executable/featurecounts/.config.vsh.yaml index 1c8726b5..a15f1bbd 100644 --- a/target/executable/featurecounts/.config.vsh.yaml +++ b/target/executable/featurecounts/.config.vsh.yaml @@ -644,15 +644,15 @@ build_info: engine: "docker|native" output: "target/executable/featurecounts" executable: "target/executable/featurecounts/featurecounts" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/featurecounts/featurecounts b/target/executable/featurecounts/featurecounts index 1ec026f1..bfbd11e1 100755 --- a/target/executable/featurecounts/featurecounts +++ b/target/executable/featurecounts/featurecounts @@ -2,7 +2,7 @@ # featurecounts main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -57,24 +57,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -494,9 +494,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -548,9 +548,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -570,7 +570,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -594,9 +595,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -753,9 +754,9 @@ RUN featureCounts -v 2>&1 | sed 's/featureCounts v\([0-9.]*\)/featureCounts: \1/ LABEL org.opencontainers.image.authors="Sai Nirmayi Yasa" LABEL org.opencontainers.image.description="Companion container for running component featurecounts" -LABEL org.opencontainers.image.created="2024-08-21T11:35:57Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:45Z" LABEL org.opencontainers.image.source="https://github.com/ShiLab-Bioinformatics/subread" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -782,6 +783,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -812,14 +816,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -827,7 +834,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -835,7 +842,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -843,17 +852,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -1417,6 +1431,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -1478,6 +1504,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/gffread/.config.vsh.yaml b/target/executable/gffread/.config.vsh.yaml index 99048d86..4482e1dc 100644 --- a/target/executable/gffread/.config.vsh.yaml +++ b/target/executable/gffread/.config.vsh.yaml @@ -684,15 +684,15 @@ build_info: engine: "docker|native" output: "target/executable/gffread" executable: "target/executable/gffread/gffread" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/gffread/gffread b/target/executable/gffread/gffread index 68fa70fd..82249d59 100755 --- a/target/executable/gffread/gffread +++ b/target/executable/gffread/gffread @@ -2,7 +2,7 @@ # gffread main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -57,24 +57,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -547,9 +547,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -601,9 +601,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -623,7 +623,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -647,9 +648,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -806,9 +807,9 @@ RUN echo "gffread: \"$(gffread --version 2>&1)\"" > /var/software_versions.txt LABEL org.opencontainers.image.authors="Emma Rousseau" LABEL org.opencontainers.image.description="Companion container for running component gffread" -LABEL org.opencontainers.image.created="2024-08-21T11:35:49Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:37Z" LABEL org.opencontainers.image.source="https://github.com/gpertea/gffread" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -835,6 +836,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -865,14 +869,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -880,7 +887,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -888,7 +895,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -896,17 +905,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -1605,6 +1619,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -1666,6 +1692,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/lofreq/lofreq_call/.config.vsh.yaml b/target/executable/lofreq/lofreq_call/.config.vsh.yaml index 82841437..6d98ffc9 100644 --- a/target/executable/lofreq/lofreq_call/.config.vsh.yaml +++ b/target/executable/lofreq/lofreq_call/.config.vsh.yaml @@ -506,15 +506,15 @@ build_info: engine: "docker|native" output: "target/executable/lofreq/lofreq_call" executable: "target/executable/lofreq/lofreq_call/lofreq_call" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/lofreq/lofreq_call/lofreq_call b/target/executable/lofreq/lofreq_call/lofreq_call index 478f8de3..ff1d3590 100755 --- a/target/executable/lofreq/lofreq_call/lofreq_call +++ b/target/executable/lofreq/lofreq_call/lofreq_call @@ -2,7 +2,7 @@ # lofreq_call main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -57,24 +57,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -395,9 +395,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -449,9 +449,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -471,7 +471,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -495,9 +496,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -655,9 +656,9 @@ echo "lofreq: $version" > /var/software_versions.txt LABEL org.opencontainers.image.authors="Kai Waldrant" LABEL org.opencontainers.image.description="Companion container for running component lofreq lofreq_call" -LABEL org.opencontainers.image.created="2024-08-21T11:35:57Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:45Z" LABEL org.opencontainers.image.source="https://github.com/viash-hub/biobox" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -684,6 +685,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -714,14 +718,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -729,7 +736,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -737,7 +744,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -745,17 +754,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -1239,6 +1253,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -1300,6 +1326,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/lofreq/lofreq_indelqual/.config.vsh.yaml b/target/executable/lofreq/lofreq_indelqual/.config.vsh.yaml index bdaa51fa..07759904 100644 --- a/target/executable/lofreq/lofreq_indelqual/.config.vsh.yaml +++ b/target/executable/lofreq/lofreq_indelqual/.config.vsh.yaml @@ -214,15 +214,15 @@ build_info: engine: "docker|native" output: "target/executable/lofreq/lofreq_indelqual" executable: "target/executable/lofreq/lofreq_indelqual/lofreq_indelqual" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/lofreq/lofreq_indelqual/lofreq_indelqual b/target/executable/lofreq/lofreq_indelqual/lofreq_indelqual index b46d4602..2c330755 100755 --- a/target/executable/lofreq/lofreq_indelqual/lofreq_indelqual +++ b/target/executable/lofreq/lofreq_indelqual/lofreq_indelqual @@ -2,7 +2,7 @@ # lofreq_indelqual main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -57,24 +57,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -240,9 +240,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -294,9 +294,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -316,7 +316,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -340,9 +341,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -500,9 +501,9 @@ echo "lofreq: $version" > /var/software_versions.txt LABEL org.opencontainers.image.authors="Kai Waldrant" LABEL org.opencontainers.image.description="Companion container for running component lofreq lofreq_indelqual" -LABEL org.opencontainers.image.created="2024-08-21T11:35:57Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:45Z" LABEL org.opencontainers.image.source="https://github.com/viash-hub/biobox" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -529,6 +530,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -559,14 +563,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -574,7 +581,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -582,7 +589,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -590,17 +599,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -723,6 +737,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -784,6 +810,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/multiqc/.config.vsh.yaml b/target/executable/multiqc/.config.vsh.yaml index 0d097e6d..98e8b7d2 100644 --- a/target/executable/multiqc/.config.vsh.yaml +++ b/target/executable/multiqc/.config.vsh.yaml @@ -455,15 +455,15 @@ build_info: engine: "docker|native" output: "target/executable/multiqc" executable: "target/executable/multiqc/multiqc" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/multiqc/multiqc b/target/executable/multiqc/multiqc index 4b874ef1..9b2dc160 100755 --- a/target/executable/multiqc/multiqc +++ b/target/executable/multiqc/multiqc @@ -2,7 +2,7 @@ # multiqc main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -57,24 +57,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -377,9 +377,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -431,9 +431,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -453,7 +453,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -477,9 +478,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -636,9 +637,9 @@ RUN multiqc --version | sed 's/multiqc, version\s\(.*\)/multiqc: "\1"/' > /var/s LABEL org.opencontainers.image.authors="Dorien Roosen" LABEL org.opencontainers.image.description="Companion container for running component multiqc" -LABEL org.opencontainers.image.created="2024-08-21T11:36:04Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:53Z" LABEL org.opencontainers.image.source="https://github.com/viash-hub/biobox" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -665,6 +666,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -695,14 +699,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -710,7 +717,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -718,7 +725,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -726,17 +735,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -1105,6 +1119,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -1166,6 +1192,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/pear/.config.vsh.yaml b/target/executable/pear/.config.vsh.yaml index 1e440ada..5ef2a52e 100644 --- a/target/executable/pear/.config.vsh.yaml +++ b/target/executable/pear/.config.vsh.yaml @@ -397,15 +397,15 @@ build_info: engine: "docker|native" output: "target/executable/pear" executable: "target/executable/pear/pear" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/pear/pear b/target/executable/pear/pear index f39adbd6..87bbe61d 100755 --- a/target/executable/pear/pear +++ b/target/executable/pear/pear @@ -2,7 +2,7 @@ # pear main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -57,24 +57,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -336,9 +336,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -390,9 +390,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -412,7 +412,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -436,9 +437,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -596,9 +597,9 @@ echo "pear: $version" > /var/software_versions.txt LABEL org.opencontainers.image.authors="Kai Waldrant" LABEL org.opencontainers.image.description="Companion container for running component pear" -LABEL org.opencontainers.image.created="2024-08-21T11:35:52Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:40Z" LABEL org.opencontainers.image.source="https://github.com/tseemann/PEAR" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -625,6 +626,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -655,14 +659,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -670,7 +677,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -678,7 +685,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -686,17 +695,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -1032,6 +1046,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -1093,6 +1119,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/qualimap/qualimap_rnaseq/.config.vsh.yaml b/target/executable/qualimap/qualimap_rnaseq/.config.vsh.yaml index d51b3b66..2bba5e90 100644 --- a/target/executable/qualimap/qualimap_rnaseq/.config.vsh.yaml +++ b/target/executable/qualimap/qualimap_rnaseq/.config.vsh.yaml @@ -263,15 +263,15 @@ build_info: engine: "docker|native" output: "target/executable/qualimap/qualimap_rnaseq" executable: "target/executable/qualimap/qualimap_rnaseq/qualimap_rnaseq" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/qualimap/qualimap_rnaseq/qualimap_rnaseq b/target/executable/qualimap/qualimap_rnaseq/qualimap_rnaseq index d7d58052..92558344 100755 --- a/target/executable/qualimap/qualimap_rnaseq/qualimap_rnaseq +++ b/target/executable/qualimap/qualimap_rnaseq/qualimap_rnaseq @@ -2,7 +2,7 @@ # qualimap_rnaseq main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -57,24 +57,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -267,9 +267,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -321,9 +321,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -343,7 +343,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -367,9 +368,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -526,9 +527,9 @@ RUN echo QualiMap: $(qualimap 2>&1 | grep QualiMap | sed 's/^.*QualiMap//') > /v LABEL org.opencontainers.image.authors="Dorien Roosen" LABEL org.opencontainers.image.description="Companion container for running component qualimap qualimap_rnaseq" -LABEL org.opencontainers.image.created="2024-08-21T11:35:55Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:44Z" LABEL org.opencontainers.image.source="https://bitbucket.org/kokonech/qualimap/commits/branch/master" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -555,6 +556,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -585,14 +589,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -600,7 +607,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -608,7 +615,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -616,17 +625,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -797,6 +811,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -858,6 +884,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/rsem/rsem_prepare_reference/.config.vsh.yaml b/target/executable/rsem/rsem_prepare_reference/.config.vsh.yaml new file mode 100644 index 00000000..52ca8fce --- /dev/null +++ b/target/executable/rsem/rsem_prepare_reference/.config.vsh.yaml @@ -0,0 +1,442 @@ +name: "rsem_prepare_reference" +namespace: "rsem" +version: "main" +authors: +- name: "Sai Nirmayi Yasa" + roles: + - "author" + - "maintainer" + info: + links: + email: "nirmayi@data-intuitive.com" + github: "sainirmayi" + linkedin: "sai-nirmayi-yasa" + organizations: + - name: "Data Intuitive" + href: "https://www.data-intuitive.com" + role: "Junior Bioinformatics Researcher" +argument_groups: +- name: "Inputs" + arguments: + - type: "file" + name: "--reference_fasta_files" + description: "Semi-colon separated list of Multi-FASTA formatted files OR a directory\ + \ name. If a directory name is specified, RSEM will read all files with suffix\ + \ \".fa\" or \".fasta\" in this directory. The files should contain either the\ + \ sequences of transcripts or an entire genome, depending on whether the '--gtf'\ + \ option is used.\n" + info: null + example: + - "read1.fasta" + must_exist: true + create_parent: true + required: true + direction: "input" + multiple: true + multiple_sep: ";" + - type: "string" + name: "--reference_name" + description: "The name of the reference used. RSEM will generate several reference-related\ + \ files that are prefixed by this name. This name can contain path information\ + \ (e.g. '/ref/mm9').\n" + info: null + example: + - "/ref/mm9" + required: true + direction: "input" + multiple: false + multiple_sep: ";" +- name: "Outputs" + arguments: + - type: "file" + name: "--output" + description: "Directory containing reference files generated by RSEM." + info: null + must_exist: true + create_parent: true + required: true + direction: "output" + multiple: false + multiple_sep: ";" +- name: "Other options" + arguments: + - type: "file" + name: "--gtf" + description: "Assume that 'reference_fasta_files' contains the sequence of a genome,\ + \ and extract transcript reference sequences using the gene annotations specified\ + \ in the GTF file. If this and '--gff3' options are not provided, RSEM will\ + \ assume 'reference_fasta_files' contains the reference transcripts. In this\ + \ case, RSEM assumes that name of each sequence in the Multi-FASTA files is\ + \ its transcript_id." + info: null + example: + - "annotations.gtf" + must_exist: true + create_parent: true + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "file" + name: "--gff3" + description: "GFF3 annotation file. Converted to GTF format with the file name\ + \ 'reference_name.gtf'. Please make sure that 'reference_name.gtf' does not\ + \ exist." + info: null + example: + - "annotations.gff" + must_exist: true + create_parent: true + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "string" + name: "--gff3_rna_patterns" + description: "List of transcript categories (separated by semi-colon). Only transcripts\ + \ that match the string will be extracted." + info: null + example: + - "mRNA;rRNA" + required: false + direction: "input" + multiple: true + multiple_sep: ";" + - type: "boolean_true" + name: "--gff3_genes_as_transcripts" + description: "This option is designed for untypical organisms, such as viruses,\ + \ whose GFF3 files only contain genes. RSEM will assume each gene as a unique\ + \ transcript when it converts the GFF3 file into GTF format." + info: null + direction: "input" + - type: "string" + name: "--trusted_sources" + description: "List of trusted sources (separated by semi-colon). Only transcripts\ + \ coming from these sources will be extracted. If this option is off, all sources\ + \ are accepted." + info: null + example: + - "ENSEMBL;HAVANA" + required: false + direction: "input" + multiple: true + multiple_sep: ";" + - type: "file" + name: "--transcript_to_gene_map" + description: "Use information from this file to map from transcript (isoform)\ + \ ids to gene ids. Each line of this file should be of the form: \n gene_id\ + \ transcript_id\nwith the two fields separated by a tab character.\nIf you are\ + \ using a GTF file for the \"UCSC Genes\" gene set from the UCSC Genome Browser,\ + \ then the \"knownIsoforms.txt\" file (obtained from the \"Downloads\" section\ + \ of the UCSC Genome Browser site) is of this format. \nIf this option is off,\ + \ then the mapping of isoforms to genes depends on whether the '--gtf' option\ + \ is specified. If '--gtf' is specified, then RSEM uses the \"gene_id\" and\ + \ \"transcript_id\" attributes in the GTF file. Otherwise, RSEM assumes that\ + \ each sequence in the reference sequence files is a separate gene.\n" + info: null + example: + - "isoforms.txt" + must_exist: true + create_parent: true + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "file" + name: "--allele_to_gene_map" + description: "Use information from to provide gene_id and transcript_id\ + \ information for each allele-specific transcript. Each line of should\ + \ be of the form:\n gene_id transcript_id allele_id\nwith the fields separated\ + \ by a tab character.\nThis option is designed for quantifying allele-specific\ + \ expression. It is only valid if '--gtf' option is not specified. allele_id\ + \ should be the sequence names presented in the Multi-FASTA-formatted files.\n" + info: null + must_exist: true + create_parent: true + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "boolean_true" + name: "--polyA" + description: "Add poly(A) tails to the end of all reference isoforms. The length\ + \ of poly(A) tail added is specified by '--polyA-length' option. STAR aligner\ + \ users may not want to use this option." + info: null + direction: "input" + - type: "integer" + name: "--polyA_length" + description: "The length of the poly(A) tails to be added." + info: null + example: + - 125 + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "file" + name: "--no_polyA_subset" + description: "Only meaningful if '--polyA' is specified. Do not add poly(A) tails\ + \ to those transcripts listed in this file containing a list of transcript_ids." + info: null + example: + - "transcript_ids.txt" + must_exist: true + create_parent: true + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "boolean_true" + name: "--bowtie" + description: "Build Bowtie indices." + info: null + direction: "input" + - type: "boolean_true" + name: "--bowtie2" + description: "Build Bowtie 2 indices." + info: null + direction: "input" + - type: "boolean_true" + name: "--star" + description: "Build STAR indices." + info: null + direction: "input" + - type: "integer" + name: "--star_sjdboverhang" + description: "Length of the genomic sequence around annotated junction. It is\ + \ only used for STAR to build splice junctions database and not needed for Bowtie\ + \ or Bowtie2. It will be passed as the --sjdbOverhang option to STAR. According\ + \ to STAR's manual, its ideal value is max(ReadLength)-1, e.g. for 2x101 paired-end\ + \ reads, the ideal value is 101-1=100. In most cases, the default value of 100\ + \ will work as well as the ideal value. (Default is 100)" + info: null + example: + - 100 + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "boolean_true" + name: "--hisat2_hca" + description: "Build HISAT2 indices on the transcriptome according to Human Cell\ + \ Atlas (HCA) SMART-Seq2 pipeline." + info: null + direction: "input" + - type: "boolean_true" + name: "--quiet" + alternatives: + - "-q" + description: "Suppress the output of logging information." + info: null + direction: "input" +- name: "Prior-enhanced RSEM options" + arguments: + - type: "boolean_true" + name: "--prep_pRSEM" + description: "A Boolean indicating whether to prepare reference files for pRSEM,\ + \ including building Bowtie indices for a genome and selecting training set\ + \ isoforms. The index files will be used for aligning ChIP-seq reads in prior-enhanced\ + \ RSEM and the training set isoforms will be used for learning prior. A path\ + \ to Bowtie executables and a mappability file in bigWig format are required\ + \ when this option is on. Currently, Bowtie2 is not supported for prior-enhanced\ + \ RSEM." + info: null + direction: "input" + - type: "file" + name: "--mappability_bigwig_file" + description: "Full path to a whole-genome mappability file in bigWig format. This\ + \ file is required for running prior-enhanced RSEM. It is used for selecting\ + \ a training set of isoforms for prior-learning. This file can be either downloaded\ + \ from UCSC Genome Browser or generated by GEM (Derrien et al., 2012, PLoS One)." + info: null + must_exist: true + create_parent: true + required: false + direction: "input" + multiple: false + multiple_sep: ";" +resources: +- type: "bash_script" + path: "script.sh" + is_executable: true +description: "RSEM is a software package for estimating gene and isoform expression\ + \ levels from RNA-Seq data. This component prepares transcript references for RSEM.\n" +test_resources: +- type: "bash_script" + path: "test.sh" + is_executable: true +info: null +status: "enabled" +requirements: + commands: + - "ps" +keywords: +- "Transcriptome" +- "Index" +license: "GPL-3.0" +references: + doi: + - "10.1186/1471-2105-12-323" +links: + repository: "https://github.com/deweylab/RSEM" + homepage: "http://deweylab.github.io/RSEM" + documentation: "https://deweylab.github.io/RSEM/rsem-prepare-reference.html" +runners: +- type: "executable" + id: "executable" + docker_setup_strategy: "ifneedbepullelsecachedbuild" +- type: "nextflow" + id: "nextflow" + directives: + tag: "$id" + auto: + simplifyInput: true + simplifyOutput: false + transcript: false + publish: false + config: + labels: + mem1gb: "memory = 1000000000.B" + mem2gb: "memory = 2000000000.B" + mem5gb: "memory = 5000000000.B" + mem10gb: "memory = 10000000000.B" + mem20gb: "memory = 20000000000.B" + mem50gb: "memory = 50000000000.B" + mem100gb: "memory = 100000000000.B" + mem200gb: "memory = 200000000000.B" + mem500gb: "memory = 500000000000.B" + mem1tb: "memory = 1000000000000.B" + mem2tb: "memory = 2000000000000.B" + mem5tb: "memory = 5000000000000.B" + mem10tb: "memory = 10000000000000.B" + mem20tb: "memory = 20000000000000.B" + mem50tb: "memory = 50000000000000.B" + mem100tb: "memory = 100000000000000.B" + mem200tb: "memory = 200000000000000.B" + mem500tb: "memory = 500000000000000.B" + mem1gib: "memory = 1073741824.B" + mem2gib: "memory = 2147483648.B" + mem4gib: "memory = 4294967296.B" + mem8gib: "memory = 8589934592.B" + mem16gib: "memory = 17179869184.B" + mem32gib: "memory = 34359738368.B" + mem64gib: "memory = 68719476736.B" + mem128gib: "memory = 137438953472.B" + mem256gib: "memory = 274877906944.B" + mem512gib: "memory = 549755813888.B" + mem1tib: "memory = 1099511627776.B" + mem2tib: "memory = 2199023255552.B" + mem4tib: "memory = 4398046511104.B" + mem8tib: "memory = 8796093022208.B" + mem16tib: "memory = 17592186044416.B" + mem32tib: "memory = 35184372088832.B" + mem64tib: "memory = 70368744177664.B" + mem128tib: "memory = 140737488355328.B" + mem256tib: "memory = 281474976710656.B" + mem512tib: "memory = 562949953421312.B" + cpu1: "cpus = 1" + cpu2: "cpus = 2" + cpu5: "cpus = 5" + cpu10: "cpus = 10" + cpu20: "cpus = 20" + cpu50: "cpus = 50" + cpu100: "cpus = 100" + cpu200: "cpus = 200" + cpu500: "cpus = 500" + cpu1000: "cpus = 1000" + debug: false + container: "docker" +engines: +- type: "docker" + id: "docker" + image: "ubuntu:22.04" + target_registry: "images.viash-hub.com" + target_tag: "main" + namespace_separator: "/" + setup: + - type: "apt" + packages: + - "build-essential" + - "gcc" + - "g++" + - "make" + - "wget" + - "zlib1g-dev" + - "unzip xxd" + - "perl" + - "r-base" + - "bowtie2" + - "pip" + - "git" + interactive: false + - type: "python" + user: false + packages: + - "bowtie" + upgrade: true + - type: "docker" + run: + - "ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone\ + \ && \\\ncd /tmp && \\\nwget --no-check-certificate https://github.com/alexdobin/STAR/archive/refs/tags/${STAR_VERSION}.zip\ + \ && \\\nunzip ${STAR_VERSION}.zip && \\\ncd STAR-${STAR_VERSION}/source &&\ + \ \\\nmake STARstatic CXXFLAGS_SIMD=-std=c++11 && \\\ncp STAR /usr/local/bin\ + \ && \\\ncd /tmp && \\\nwget --no-check-certificate https://github.com/deweylab/RSEM/archive/refs/tags/v${RSEM_VERSION}.zip\ + \ && \\\nunzip v${RSEM_VERSION}.zip && \\\ncd RSEM-${RSEM_VERSION} && \\\nmake\ + \ && \\\nmake install && \\\ncd /tmp && \\\nwget --no-check-certificate -O bowtie-${BOWTIE_VERSION}-linux-x86_64.zip\ + \ https://sourceforge.net/projects/bowtie-bio/files/bowtie/${BOWTIE_VERSION}/bowtie-${BOWTIE_VERSION}-linux-x86_64.zip/download\ + \ && \\\nunzip bowtie-${BOWTIE_VERSION}-linux-x86_64.zip && \\\ncp bowtie-${BOWTIE_VERSION}-linux-x86_64/bowtie*\ + \ /usr/local/bin && \\\ncd /tmp && \\\ngit clone https://github.com/DaehwanKimLab/hisat2.git\ + \ /tmp/hisat2 && \\\ncd /tmp/hisat2 && \\\nmake && \\\ncp -r hisat2* /usr/local/bin\ + \ && \\\ncd && \\\nrm -rf /tmp/STAR-${STAR_VERSION} /tmp/${STAR_VERSION}.zip\ + \ /tmp/bowtie-${BOWTIE_VERSION}-linux-x86_64 /tmp/hisat2 && \\\napt-get --purge\ + \ autoremove -y ${PACKAGES} && \\\napt-get clean \n" + env: + - "STAR_VERSION=2.7.11b" + - "RSEM_VERSION=1.3.3" + - "BOWTIE_VERSION=1.3.1" + - "TZ=Europe/Brussels" + - type: "docker" + run: + - "echo \"RSEM: `rsem-calculate-expression --version | sed -e 's/Current version:\ + \ RSEM v//g'`\" > /var/software_versions.txt && \\\necho \"STAR: `STAR --version`\"\ + \ >> /var/software_versions.txt && \\\necho \"bowtie2: `bowtie2 --version |\ + \ grep -oP '\\d+\\.\\d+\\.\\d+'`\" >> /var/software_versions.txt && \\\necho\ + \ \"bowtie: `bowtie --version | grep -oP 'bowtie-align-s version \\K\\d+\\.\\\ + d+\\.\\d+'`\" >> /var/software_versions.txt && \\\necho \"HISAT2: `hisat2 --version\ + \ | grep -oP 'hisat2-align-s version \\K\\d+\\.\\d+\\.\\d+'`\" >> /var/software_versions.txt\n" + entrypoint: [] + cmd: null +- type: "native" + id: "native" +build_info: + config: "src/rsem/rsem_prepare_reference/config.vsh.yaml" + runner: "executable" + engine: "docker|native" + output: "target/executable/rsem/rsem_prepare_reference" + executable: "target/executable/rsem/rsem_prepare_reference/rsem_prepare_reference" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" + git_remote: "https://github.com/viash-hub/biobox" +package_config: + name: "biobox" + version: "main" + description: "A collection of bioinformatics tools for working with sequence data.\n" + info: null + viash_version: "0.9.0-RC7" + source: "src" + target: "target" + config_mods: + - ".requirements.commands := ['ps']\n" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'main'" + keywords: + - "bioinformatics" + - "modules" + - "sequencing" + license: "MIT" + organization: "vsh" + links: + repository: "https://github.com/viash-hub/biobox" + issue_tracker: "https://github.com/viash-hub/biobox/issues" diff --git a/target/executable/rsem/rsem_prepare_reference/rsem_prepare_reference b/target/executable/rsem/rsem_prepare_reference/rsem_prepare_reference new file mode 100755 index 00000000..55877f1e --- /dev/null +++ b/target/executable/rsem/rsem_prepare_reference/rsem_prepare_reference @@ -0,0 +1,1689 @@ +#!/usr/bin/env bash + +# rsem_prepare_reference main +# +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a +# derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from +# Data Intuitive. +# +# The component may contain files which fall under a different license. The +# authors of this component should specify the license in the header of such +# files, or include a separate license file detailing the licenses of all included +# files. +# +# Component authors: +# * Sai Nirmayi Yasa (author, maintainer) + +set -e + +if [ -z "$VIASH_TEMP" ]; then + VIASH_TEMP=${VIASH_TEMP:-$VIASH_TMPDIR} + VIASH_TEMP=${VIASH_TEMP:-$VIASH_TEMPDIR} + VIASH_TEMP=${VIASH_TEMP:-$VIASH_TMP} + VIASH_TEMP=${VIASH_TEMP:-$TMPDIR} + VIASH_TEMP=${VIASH_TEMP:-$TMP} + VIASH_TEMP=${VIASH_TEMP:-$TEMPDIR} + VIASH_TEMP=${VIASH_TEMP:-$TEMP} + VIASH_TEMP=${VIASH_TEMP:-/tmp} +fi + +# define helper functions +# ViashQuote: put quotes around non flag values +# $1 : unquoted string +# return : possibly quoted string +# examples: +# ViashQuote --foo # returns --foo +# ViashQuote bar # returns 'bar' +# Viashquote --foo=bar # returns --foo='bar' +function ViashQuote { + if [[ "$1" =~ ^-+[a-zA-Z0-9_\-]+=.+$ ]]; then + echo "$1" | sed "s#=\(.*\)#='\1'#" + elif [[ "$1" =~ ^-+[a-zA-Z0-9_\-]+$ ]]; then + echo "$1" + else + echo "'$1'" + fi +} +# ViashRemoveFlags: Remove leading flag +# $1 : string with a possible leading flag +# return : string without possible leading flag +# examples: +# ViashRemoveFlags --foo=bar # returns bar +function ViashRemoveFlags { + echo "$1" | sed 's/^--*[a-zA-Z0-9_\-]*=//' +} +# ViashSourceDir: return the path of a bash file, following symlinks +# usage : ViashSourceDir ${BASH_SOURCE[0]} +# $1 : Should always be set to ${BASH_SOURCE[0]} +# returns : The absolute path of the bash file +function ViashSourceDir { + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" + done + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd +} +# ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks +# usage : ViashFindTargetDir 'ScriptPath' +# $1 : The location from where to start the upward search +# returns : The absolute path of the '.build.yaml' file +function ViashFindTargetDir { + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} + done + echo $source +} +# see https://en.wikipedia.org/wiki/Syslog#Severity_level +VIASH_LOGCODE_EMERGENCY=0 +VIASH_LOGCODE_ALERT=1 +VIASH_LOGCODE_CRITICAL=2 +VIASH_LOGCODE_ERROR=3 +VIASH_LOGCODE_WARNING=4 +VIASH_LOGCODE_NOTICE=5 +VIASH_LOGCODE_INFO=6 +VIASH_LOGCODE_DEBUG=7 +VIASH_VERBOSITY=$VIASH_LOGCODE_NOTICE + +# ViashLog: Log events depending on the verbosity level +# usage: ViashLog 1 alert Oh no something went wrong! +# $1: required verbosity level +# $2: display tag +# $3+: messages to display +# stdout: Your input, prepended by '[$2] '. +function ViashLog { + local required_level="$1" + local display_tag="$2" + shift 2 + if [ $VIASH_VERBOSITY -ge $required_level ]; then + >&2 echo "[$display_tag]" "$@" + fi +} + +# ViashEmergency: log events when the system is unstable +# usage: ViashEmergency Oh no something went wrong. +# stdout: Your input, prepended by '[emergency] '. +function ViashEmergency { + ViashLog $VIASH_LOGCODE_EMERGENCY emergency "$@" +} + +# ViashAlert: log events when actions must be taken immediately (e.g. corrupted system database) +# usage: ViashAlert Oh no something went wrong. +# stdout: Your input, prepended by '[alert] '. +function ViashAlert { + ViashLog $VIASH_LOGCODE_ALERT alert "$@" +} + +# ViashCritical: log events when a critical condition occurs +# usage: ViashCritical Oh no something went wrong. +# stdout: Your input, prepended by '[critical] '. +function ViashCritical { + ViashLog $VIASH_LOGCODE_CRITICAL critical "$@" +} + +# ViashError: log events when an error condition occurs +# usage: ViashError Oh no something went wrong. +# stdout: Your input, prepended by '[error] '. +function ViashError { + ViashLog $VIASH_LOGCODE_ERROR error "$@" +} + +# ViashWarning: log potentially abnormal events +# usage: ViashWarning Something may have gone wrong. +# stdout: Your input, prepended by '[warning] '. +function ViashWarning { + ViashLog $VIASH_LOGCODE_WARNING warning "$@" +} + +# ViashNotice: log significant but normal events +# usage: ViashNotice This just happened. +# stdout: Your input, prepended by '[notice] '. +function ViashNotice { + ViashLog $VIASH_LOGCODE_NOTICE notice "$@" +} + +# ViashInfo: log normal events +# usage: ViashInfo This just happened. +# stdout: Your input, prepended by '[info] '. +function ViashInfo { + ViashLog $VIASH_LOGCODE_INFO info "$@" +} + +# ViashDebug: log all events, for debugging purposes +# usage: ViashDebug This just happened. +# stdout: Your input, prepended by '[debug] '. +function ViashDebug { + ViashLog $VIASH_LOGCODE_DEBUG debug "$@" +} + +# find source folder of this component +VIASH_META_RESOURCES_DIR=`ViashSourceDir ${BASH_SOURCE[0]}` + +# find the root of the built components & dependencies +VIASH_TARGET_DIR=`ViashFindTargetDir $VIASH_META_RESOURCES_DIR` + +# define meta fields +VIASH_META_NAME="rsem_prepare_reference" +VIASH_META_FUNCTIONALITY_NAME="rsem_prepare_reference" +VIASH_META_EXECUTABLE="$VIASH_META_RESOURCES_DIR/$VIASH_META_NAME" +VIASH_META_CONFIG="$VIASH_META_RESOURCES_DIR/.config.vsh.yaml" +VIASH_META_TEMP_DIR="$VIASH_TEMP" + + +# ViashHelp: Display helpful explanation about this executable +function ViashHelp { + echo "rsem_prepare_reference main" + echo "" + echo "RSEM is a software package for estimating gene and isoform expression levels" + echo "from RNA-Seq data. This component prepares transcript references for RSEM." + echo "" + echo "Inputs:" + echo " --reference_fasta_files" + echo " type: file, required parameter, multiple values allowed, file must exist" + echo " example: read1.fasta" + echo " Semi-colon separated list of Multi-FASTA formatted files OR a directory" + echo " name. If a directory name is specified, RSEM will read all files with" + echo " suffix \".fa\" or \".fasta\" in this directory. The files should contain" + echo " either the sequences of transcripts or an entire genome, depending on" + echo " whether the '--gtf' option is used." + echo "" + echo " --reference_name" + echo " type: string, required parameter" + echo " example: /ref/mm9" + echo " The name of the reference used. RSEM will generate several" + echo " reference-related files that are prefixed by this name. This name can" + echo " contain path information (e.g. '/ref/mm9')." + echo "" + echo "Outputs:" + echo " --output" + echo " type: file, required parameter, output, file must exist" + echo " Directory containing reference files generated by RSEM." + echo "" + echo "Other options:" + echo " --gtf" + echo " type: file, file must exist" + echo " example: annotations.gtf" + echo " Assume that 'reference_fasta_files' contains the sequence of a genome," + echo " and extract transcript reference sequences using the gene annotations" + echo " specified in the GTF file. If this and '--gff3' options are not" + echo " provided, RSEM will assume 'reference_fasta_files' contains the" + echo " reference transcripts. In this case, RSEM assumes that name of each" + echo " sequence in the Multi-FASTA files is its transcript_id." + echo "" + echo " --gff3" + echo " type: file, file must exist" + echo " example: annotations.gff" + echo " GFF3 annotation file. Converted to GTF format with the file name" + echo " 'reference_name.gtf'. Please make sure that 'reference_name.gtf' does" + echo " not exist." + echo "" + echo " --gff3_rna_patterns" + echo " type: string, multiple values allowed" + echo " example: mRNA;rRNA" + echo " List of transcript categories (separated by semi-colon). Only" + echo " transcripts that match the string will be extracted." + echo "" + echo " --gff3_genes_as_transcripts" + echo " type: boolean_true" + echo " This option is designed for untypical organisms, such as viruses, whose" + echo " GFF3 files only contain genes. RSEM will assume each gene as a unique" + echo " transcript when it converts the GFF3 file into GTF format." + echo "" + echo " --trusted_sources" + echo " type: string, multiple values allowed" + echo " example: ENSEMBL;HAVANA" + echo " List of trusted sources (separated by semi-colon). Only transcripts" + echo " coming from these sources will be extracted. If this option is off, all" + echo " sources are accepted." + echo "" + echo " --transcript_to_gene_map" + echo " type: file, file must exist" + echo " example: isoforms.txt" + echo " Use information from this file to map from transcript (isoform) ids to" + echo " gene ids. Each line of this file should be of the form:" + echo " gene_id transcript_id" + echo " with the two fields separated by a tab character." + echo " If you are using a GTF file for the \"UCSC Genes\" gene set from the UCSC" + echo " Genome Browser, then the \"knownIsoforms.txt\" file (obtained from the" + echo " \"Downloads\" section of the UCSC Genome Browser site) is of this format." + echo " If this option is off, then the mapping of isoforms to genes depends on" + echo " whether the '--gtf' option is specified. If '--gtf' is specified, then" + echo " RSEM uses the \"gene_id\" and \"transcript_id\" attributes in the GTF file." + echo " Otherwise, RSEM assumes that each sequence in the reference sequence" + echo " files is a separate gene." + echo "" + echo " --allele_to_gene_map" + echo " type: file, file must exist" + echo " Use information from to provide gene_id and transcript_id" + echo " information for each allele-specific transcript. Each line of " + echo " should be of the form:" + echo " gene_id transcript_id allele_id" + echo " with the fields separated by a tab character." + echo " This option is designed for quantifying allele-specific expression. It" + echo " is only valid if '--gtf' option is not specified. allele_id should be" + echo " the sequence names presented in the Multi-FASTA-formatted files." + echo "" + echo " --polyA" + echo " type: boolean_true" + echo " Add poly(A) tails to the end of all reference isoforms. The length of" + echo " poly(A) tail added is specified by '--polyA-length' option. STAR aligner" + echo " users may not want to use this option." + echo "" + echo " --polyA_length" + echo " type: integer" + echo " example: 125" + echo " The length of the poly(A) tails to be added." + echo "" + echo " --no_polyA_subset" + echo " type: file, file must exist" + echo " example: transcript_ids.txt" + echo " Only meaningful if '--polyA' is specified. Do not add poly(A) tails to" + echo " those transcripts listed in this file containing a list of" + echo " transcript_ids." + echo "" + echo " --bowtie" + echo " type: boolean_true" + echo " Build Bowtie indices." + echo "" + echo " --bowtie2" + echo " type: boolean_true" + echo " Build Bowtie 2 indices." + echo "" + echo " --star" + echo " type: boolean_true" + echo " Build STAR indices." + echo "" + echo " --star_sjdboverhang" + echo " type: integer" + echo " example: 100" + echo " Length of the genomic sequence around annotated junction. It is only" + echo " used for STAR to build splice junctions database and not needed for" + echo " Bowtie or Bowtie2. It will be passed as the --sjdbOverhang option to" + echo " STAR. According to STAR's manual, its ideal value is max(ReadLength)-1," + echo " e.g. for 2x101 paired-end reads, the ideal value is 101-1=100. In most" + echo " cases, the default value of 100 will work as well as the ideal value." + echo " (Default is 100)" + echo "" + echo " --hisat2_hca" + echo " type: boolean_true" + echo " Build HISAT2 indices on the transcriptome according to Human Cell Atlas" + echo " (HCA) SMART-Seq2 pipeline." + echo "" + echo " -q, --quiet" + echo " type: boolean_true" + echo " Suppress the output of logging information." + echo "" + echo "Prior-enhanced RSEM options:" + echo " --prep_pRSEM" + echo " type: boolean_true" + echo " A Boolean indicating whether to prepare reference files for pRSEM," + echo " including building Bowtie indices for a genome and selecting training" + echo " set isoforms. The index files will be used for aligning ChIP-seq reads" + echo " in prior-enhanced RSEM and the training set isoforms will be used for" + echo " learning prior. A path to Bowtie executables and a mappability file in" + echo " bigWig format are required when this option is on. Currently, Bowtie2 is" + echo " not supported for prior-enhanced RSEM." + echo "" + echo " --mappability_bigwig_file" + echo " type: file, file must exist" + echo " Full path to a whole-genome mappability file in bigWig format. This file" + echo " is required for running prior-enhanced RSEM. It is used for selecting a" + echo " training set of isoforms for prior-learning. This file can be either" + echo " downloaded from UCSC Genome Browser or generated by GEM (Derrien et al.," + echo " 2012, PLoS One)." +} + +# initialise variables +VIASH_MODE='run' +VIASH_ENGINE_ID='docker' + +######## Helper functions for setting up Docker images for viash ######## +# expects: ViashDockerBuild + +# ViashDockerInstallationCheck: check whether Docker is installed correctly +# +# examples: +# ViashDockerInstallationCheck +function ViashDockerInstallationCheck { + ViashDebug "Checking whether Docker is installed" + if [ ! command -v docker &> /dev/null ]; then + ViashCritical "Docker doesn't seem to be installed. See 'https://docs.docker.com/get-docker/' for instructions." + exit 1 + fi + + ViashDebug "Checking whether the Docker daemon is running" + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? + [[ $save =~ e ]] && set -e + if [ $out -ne 0 ]; then + ViashCritical "Docker daemon does not seem to be running. Try one of the following:" + ViashCritical "- Try running 'dockerd' in the command line" + ViashCritical "- See https://docs.docker.com/config/daemon/" + exit 1 + fi +} + +# ViashDockerRemoteTagCheck: check whether a Docker image is available +# on a remote. Assumes `docker login` has been performed, if relevant. +# +# $1 : image identifier with format `[registry/]image[:tag]` +# exit code $? : whether or not the image was found +# examples: +# ViashDockerRemoteTagCheck python:latest +# echo $? # returns '0' +# ViashDockerRemoteTagCheck sdaizudceahifu +# echo $? # returns '1' +function ViashDockerRemoteTagCheck { + docker manifest inspect $1 > /dev/null 2> /dev/null +} + +# ViashDockerLocalTagCheck: check whether a Docker image is available locally +# +# $1 : image identifier with format `[registry/]image[:tag]` +# exit code $? : whether or not the image was found +# examples: +# docker pull python:latest +# ViashDockerLocalTagCheck python:latest +# echo $? # returns '0' +# ViashDockerLocalTagCheck sdaizudceahifu +# echo $? # returns '1' +function ViashDockerLocalTagCheck { + [ -n "$(docker images -q $1)" ] +} + +# ViashDockerPull: pull a Docker image +# +# $1 : image identifier with format `[registry/]image[:tag]` +# exit code $? : whether or not the image was found +# examples: +# ViashDockerPull python:latest +# echo $? # returns '0' +# ViashDockerPull sdaizudceahifu +# echo $? # returns '1' +function ViashDockerPull { + ViashNotice "Checking if Docker image is available at '$1'" + if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then + docker pull $1 && return 0 || return 1 + else + local save=$-; set +e + docker pull $1 2> /dev/null > /dev/null + local out=$? + [[ $save =~ e ]] && set -e + if [ $out -ne 0 ]; then + ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." + fi + return $out + fi +} + +# ViashDockerPush: push a Docker image +# +# $1 : image identifier with format `[registry/]image[:tag]` +# exit code $? : whether or not the image was found +# examples: +# ViashDockerPush python:latest +# echo $? # returns '0' +# ViashDockerPush sdaizudceahifu +# echo $? # returns '1' +function ViashDockerPush { + ViashNotice "Pushing image to '$1'" + local save=$-; set +e + local out + if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then + docker push $1 + out=$? + else + docker push $1 2> /dev/null > /dev/null + out=$? + fi + [[ $save =~ e ]] && set -e + if [ $out -eq 0 ]; then + ViashNotice "Container '$1' push succeeded." + else + ViashError "Container '$1' push errored. You might not be logged in or have the necessary permissions." + fi + return $out +} + +# ViashDockerPullElseBuild: pull a Docker image, else build it +# +# $1 : image identifier with format `[registry/]image[:tag]` +# ViashDockerBuild : a Bash function which builds a docker image, takes image identifier as argument. +# examples: +# ViashDockerPullElseBuild mynewcomponent +function ViashDockerPullElseBuild { + local save=$-; set +e + ViashDockerPull $1 + local out=$? + [[ $save =~ e ]] && set -e + if [ $out -ne 0 ]; then + ViashDockerBuild $@ + fi +} + +# ViashDockerSetup: create a Docker image, according to specified docker setup strategy +# +# $1 : image identifier with format `[registry/]image[:tag]` +# $2 : docker setup strategy, see DockerSetupStrategy.scala +# examples: +# ViashDockerSetup mynewcomponent alwaysbuild +function ViashDockerSetup { + local image_id="$1" + local setup_strategy="$2" + if [ "$setup_strategy" == "alwaysbuild" -o "$setup_strategy" == "build" -o "$setup_strategy" == "b" ]; then + ViashDockerBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "alwayspull" -o "$setup_strategy" == "pull" -o "$setup_strategy" == "p" ]; then + ViashDockerPull $image_id + elif [ "$setup_strategy" == "alwayspullelsebuild" -o "$setup_strategy" == "pullelsebuild" ]; then + ViashDockerPullElseBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "alwayspullelsecachedbuild" -o "$setup_strategy" == "pullelsecachedbuild" ]; then + ViashDockerPullElseBuild $image_id $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "alwayscachedbuild" -o "$setup_strategy" == "cachedbuild" -o "$setup_strategy" == "cb" ]; then + ViashDockerBuild $image_id $(ViashDockerBuildArgs "$engine_id") + elif [[ "$setup_strategy" =~ ^ifneedbe ]]; then + local save=$-; set +e + ViashDockerLocalTagCheck $image_id + local outCheck=$? + [[ $save =~ e ]] && set -e + if [ $outCheck -eq 0 ]; then + ViashInfo "Image $image_id already exists" + elif [ "$setup_strategy" == "ifneedbebuild" ]; then + ViashDockerBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "ifneedbecachedbuild" ]; then + ViashDockerBuild $image_id $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "ifneedbepull" ]; then + ViashDockerPull $image_id + elif [ "$setup_strategy" == "ifneedbepullelsebuild" ]; then + ViashDockerPullElseBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "ifneedbepullelsecachedbuild" ]; then + ViashDockerPullElseBuild $image_id $(ViashDockerBuildArgs "$engine_id") + else + ViashError "Unrecognised Docker strategy: $setup_strategy" + exit 1 + fi + elif [ "$setup_strategy" == "push" -o "$setup_strategy" == "forcepush" -o "$setup_strategy" == "alwayspush" ]; then + ViashDockerPush "$image_id" + elif [ "$setup_strategy" == "pushifnotpresent" -o "$setup_strategy" == "gentlepush" -o "$setup_strategy" == "maybepush" ]; then + local save=$-; set +e + ViashDockerRemoteTagCheck $image_id + local outCheck=$? + [[ $save =~ e ]] && set -e + if [ $outCheck -eq 0 ]; then + ViashNotice "Container '$image_id' exists, doing nothing." + else + ViashNotice "Container '$image_id' does not yet exist." + ViashDockerPush "$image_id" + fi + elif [ "$setup_strategy" == "donothing" -o "$setup_strategy" == "meh" ]; then + ViashNotice "Skipping setup." + else + ViashError "Unrecognised Docker strategy: $setup_strategy" + exit 1 + fi +} + +# ViashDockerCheckCommands: Check whether a docker container has the required commands +# +# $1 : image identifier with format `[registry/]image[:tag]` +# $@ : commands to verify being present +# examples: +# ViashDockerCheckCommands bash:4.0 bash ps foo +function ViashDockerCheckCommands { + local image_id="$1" + shift 1 + local commands="$@" + local save=$-; set +e + local missing # mark 'missing' as local in advance, otherwise the exit code of the command will be missing and always be '0' + missing=$(docker run --rm --entrypoint=sh "$image_id" -c "for command in $commands; do command -v \$command >/dev/null 2>&1; if [ \$? -ne 0 ]; then echo \$command; exit 1; fi; done") + local outCheck=$? + [[ $save =~ e ]] && set -e + if [ $outCheck -ne 0 ]; then + ViashError "Docker container '$image_id' does not contain command '$missing'." + exit 1 + fi +} + +# ViashDockerBuild: build a docker image +# $1 : image identifier with format `[registry/]image[:tag]` +# $... : additional arguments to pass to docker build +# $VIASH_META_TEMP_DIR : temporary directory to store dockerfile & optional resources in +# $VIASH_META_NAME : name of the component +# $VIASH_META_RESOURCES_DIR : directory containing the resources +# $VIASH_VERBOSITY : verbosity level +# exit code $? : whether or not the image was built successfully +function ViashDockerBuild { + local image_id="$1" + shift 1 + + # create temporary directory to store dockerfile & optional resources in + local tmpdir=$(mktemp -d "$VIASH_META_TEMP_DIR/dockerbuild-$VIASH_META_NAME-XXXXXX") + local dockerfile="$tmpdir/Dockerfile" + function clean_up { + rm -rf "$tmpdir" + } + trap clean_up EXIT + + # store dockerfile and resources + ViashDockerfile "$VIASH_ENGINE_ID" > "$dockerfile" + + # generate the build command + local docker_build_cmd="docker build -t '$image_id' $@ '$VIASH_META_RESOURCES_DIR' -f '$dockerfile'" + + # build the container + ViashNotice "Building container '$image_id' with Dockerfile" + ViashInfo "$docker_build_cmd" + local save=$-; set +e + if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then + eval $docker_build_cmd + else + eval $docker_build_cmd &> "$tmpdir/docker_build.log" + fi + + # check exit code + local out=$? + [[ $save =~ e ]] && set -e + if [ $out -ne 0 ]; then + ViashError "Error occurred while building container '$image_id'" + if [ $VIASH_VERBOSITY -lt $VIASH_LOGCODE_INFO ]; then + ViashError "Transcript: --------------------------------" + cat "$tmpdir/docker_build.log" + ViashError "End of transcript --------------------------" + fi + exit 1 + fi +} + +######## End of helper functions for setting up Docker images for viash ######## + +# ViashDockerFile: print the dockerfile to stdout +# $1 : engine identifier +# return : dockerfile required to run this component +# examples: +# ViashDockerFile +function ViashDockerfile { + local engine_id="$1" + + if [[ "$engine_id" == "docker" ]]; then + cat << 'VIASHDOCKER' +FROM ubuntu:22.04 +ENTRYPOINT [] +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential gcc g++ make wget zlib1g-dev unzip xxd perl r-base bowtie2 pip git && \ + rm -rf /var/lib/apt/lists/* + +RUN pip install --upgrade pip && \ + pip install --upgrade --no-cache-dir "bowtie" + +ENV STAR_VERSION=2.7.11b +ENV RSEM_VERSION=1.3.3 +ENV BOWTIE_VERSION=1.3.1 +ENV TZ=Europe/Brussels +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone && \ +cd /tmp && \ +wget --no-check-certificate https://github.com/alexdobin/STAR/archive/refs/tags/${STAR_VERSION}.zip && \ +unzip ${STAR_VERSION}.zip && \ +cd STAR-${STAR_VERSION}/source && \ +make STARstatic CXXFLAGS_SIMD=-std=c++11 && \ +cp STAR /usr/local/bin && \ +cd /tmp && \ +wget --no-check-certificate https://github.com/deweylab/RSEM/archive/refs/tags/v${RSEM_VERSION}.zip && \ +unzip v${RSEM_VERSION}.zip && \ +cd RSEM-${RSEM_VERSION} && \ +make && \ +make install && \ +cd /tmp && \ +wget --no-check-certificate -O bowtie-${BOWTIE_VERSION}-linux-x86_64.zip https://sourceforge.net/projects/bowtie-bio/files/bowtie/${BOWTIE_VERSION}/bowtie-${BOWTIE_VERSION}-linux-x86_64.zip/download && \ +unzip bowtie-${BOWTIE_VERSION}-linux-x86_64.zip && \ +cp bowtie-${BOWTIE_VERSION}-linux-x86_64/bowtie* /usr/local/bin && \ +cd /tmp && \ +git clone https://github.com/DaehwanKimLab/hisat2.git /tmp/hisat2 && \ +cd /tmp/hisat2 && \ +make && \ +cp -r hisat2* /usr/local/bin && \ +cd && \ +rm -rf /tmp/STAR-${STAR_VERSION} /tmp/${STAR_VERSION}.zip /tmp/bowtie-${BOWTIE_VERSION}-linux-x86_64 /tmp/hisat2 && \ +apt-get --purge autoremove -y ${PACKAGES} && \ +apt-get clean + +RUN echo "RSEM: `rsem-calculate-expression --version | sed -e 's/Current version: RSEM v//g'`" > /var/software_versions.txt && \ +echo "STAR: `STAR --version`" >> /var/software_versions.txt && \ +echo "bowtie2: `bowtie2 --version | grep -oP '\d+\.\d+\.\d+'`" >> /var/software_versions.txt && \ +echo "bowtie: `bowtie --version | grep -oP 'bowtie-align-s version \K\d+\.\d+\.\d+'`" >> /var/software_versions.txt && \ +echo "HISAT2: `hisat2 --version | grep -oP 'hisat2-align-s version \K\d+\.\d+\.\d+'`" >> /var/software_versions.txt + +LABEL org.opencontainers.image.authors="Sai Nirmayi Yasa" +LABEL org.opencontainers.image.description="Companion container for running component rsem rsem_prepare_reference" +LABEL org.opencontainers.image.created="2024-09-02T13:02:48Z" +LABEL org.opencontainers.image.source="https://github.com/deweylab/RSEM" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" +LABEL org.opencontainers.image.version="main" + +VIASHDOCKER + fi +} + +# ViashDockerBuildArgs: return the arguments to pass to docker build +# $1 : engine identifier +# return : arguments to pass to docker build +function ViashDockerBuildArgs { + local engine_id="$1" + + if [[ "$engine_id" == "docker" ]]; then + echo "" + fi +} + +# ViashAbsolutePath: generate absolute path from relative path +# borrowed from https://stackoverflow.com/a/21951256 +# $1 : relative filename +# return : absolute path +# examples: +# ViashAbsolutePath some_file.txt # returns /path/to/some_file.txt +# ViashAbsolutePath /foo/bar/.. # returns /foo +function ViashAbsolutePath { + local thePath + local parr + local outp + local len + if [[ ! "$1" =~ ^/ ]]; then + thePath="$PWD/$1" + else + thePath="$1" + fi + echo "$thePath" | ( + IFS=/ + read -a parr + declare -a outp + for i in "${parr[@]}"; do + case "$i" in + ''|.) continue ;; + ..) + len=${#outp[@]} + if ((len==0)); then + continue + else + unset outp[$((len-1))] + fi + ;; + *) + len=${#outp[@]} + outp[$len]="$i" + ;; + esac + done + echo /"${outp[*]}" + ) +} +# ViashDockerAutodetectMount: auto configuring docker mounts from parameters +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts +# examples: +# ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' +# ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' +function ViashDockerAutodetectMount { + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name + if [ -d "$abs_path" ]; then + mount_source="$abs_path" + base_name="" + else + mount_source=`dirname "$abs_path"` + base_name=`basename "$abs_path"` + fi + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" + if [ -z "$base_name" ]; then + echo "$mount_target" + else + echo "$mount_target/$base_name" + fi +} +function ViashDockerAutodetectMountArg { + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name + if [ -d "$abs_path" ]; then + mount_source="$abs_path" + base_name="" + else + mount_source=`dirname "$abs_path"` + base_name=`basename "$abs_path"` + fi + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" + ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" + echo "--volume=\"$mount_source:$mount_target\"" +} +function ViashDockerStripAutomount { + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" +} +# initialise variables +VIASH_DIRECTORY_MOUNTS=() + +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + +# initialise docker variables +VIASH_DOCKER_RUN_ARGS=(-i --rm) + +# initialise array +VIASH_POSITIONAL_ARGS='' + +while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) + ViashHelp + exit + ;; + ---v|---verbose) + let "VIASH_VERBOSITY=VIASH_VERBOSITY+1" + shift 1 + ;; + ---verbosity) + VIASH_VERBOSITY="$2" + shift 2 + ;; + ---verbosity=*) + VIASH_VERBOSITY="$(ViashRemoveFlags "$1")" + shift 1 + ;; + --version) + echo "rsem_prepare_reference main" + exit + ;; + --reference_fasta_files) + if [ -z "$VIASH_PAR_REFERENCE_FASTA_FILES" ]; then + VIASH_PAR_REFERENCE_FASTA_FILES="$2" + else + VIASH_PAR_REFERENCE_FASTA_FILES="$VIASH_PAR_REFERENCE_FASTA_FILES;""$2" + fi + [ $# -lt 2 ] && ViashError Not enough arguments passed to --reference_fasta_files. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --reference_fasta_files=*) + if [ -z "$VIASH_PAR_REFERENCE_FASTA_FILES" ]; then + VIASH_PAR_REFERENCE_FASTA_FILES=$(ViashRemoveFlags "$1") + else + VIASH_PAR_REFERENCE_FASTA_FILES="$VIASH_PAR_REFERENCE_FASTA_FILES;"$(ViashRemoveFlags "$1") + fi + shift 1 + ;; + --reference_name) + [ -n "$VIASH_PAR_REFERENCE_NAME" ] && ViashError Bad arguments for option \'--reference_name\': \'$VIASH_PAR_REFERENCE_NAME\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_REFERENCE_NAME="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --reference_name. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --reference_name=*) + [ -n "$VIASH_PAR_REFERENCE_NAME" ] && ViashError Bad arguments for option \'--reference_name=*\': \'$VIASH_PAR_REFERENCE_NAME\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_REFERENCE_NAME=$(ViashRemoveFlags "$1") + shift 1 + ;; + --output) + [ -n "$VIASH_PAR_OUTPUT" ] && ViashError Bad arguments for option \'--output\': \'$VIASH_PAR_OUTPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_OUTPUT="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --output. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --output=*) + [ -n "$VIASH_PAR_OUTPUT" ] && ViashError Bad arguments for option \'--output=*\': \'$VIASH_PAR_OUTPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_OUTPUT=$(ViashRemoveFlags "$1") + shift 1 + ;; + --gtf) + [ -n "$VIASH_PAR_GTF" ] && ViashError Bad arguments for option \'--gtf\': \'$VIASH_PAR_GTF\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_GTF="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --gtf. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --gtf=*) + [ -n "$VIASH_PAR_GTF" ] && ViashError Bad arguments for option \'--gtf=*\': \'$VIASH_PAR_GTF\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_GTF=$(ViashRemoveFlags "$1") + shift 1 + ;; + --gff3) + [ -n "$VIASH_PAR_GFF3" ] && ViashError Bad arguments for option \'--gff3\': \'$VIASH_PAR_GFF3\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_GFF3="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --gff3. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --gff3=*) + [ -n "$VIASH_PAR_GFF3" ] && ViashError Bad arguments for option \'--gff3=*\': \'$VIASH_PAR_GFF3\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_GFF3=$(ViashRemoveFlags "$1") + shift 1 + ;; + --gff3_rna_patterns) + if [ -z "$VIASH_PAR_GFF3_RNA_PATTERNS" ]; then + VIASH_PAR_GFF3_RNA_PATTERNS="$2" + else + VIASH_PAR_GFF3_RNA_PATTERNS="$VIASH_PAR_GFF3_RNA_PATTERNS;""$2" + fi + [ $# -lt 2 ] && ViashError Not enough arguments passed to --gff3_rna_patterns. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --gff3_rna_patterns=*) + if [ -z "$VIASH_PAR_GFF3_RNA_PATTERNS" ]; then + VIASH_PAR_GFF3_RNA_PATTERNS=$(ViashRemoveFlags "$1") + else + VIASH_PAR_GFF3_RNA_PATTERNS="$VIASH_PAR_GFF3_RNA_PATTERNS;"$(ViashRemoveFlags "$1") + fi + shift 1 + ;; + --gff3_genes_as_transcripts) + [ -n "$VIASH_PAR_GFF3_GENES_AS_TRANSCRIPTS" ] && ViashError Bad arguments for option \'--gff3_genes_as_transcripts\': \'$VIASH_PAR_GFF3_GENES_AS_TRANSCRIPTS\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_GFF3_GENES_AS_TRANSCRIPTS=true + shift 1 + ;; + --trusted_sources) + if [ -z "$VIASH_PAR_TRUSTED_SOURCES" ]; then + VIASH_PAR_TRUSTED_SOURCES="$2" + else + VIASH_PAR_TRUSTED_SOURCES="$VIASH_PAR_TRUSTED_SOURCES;""$2" + fi + [ $# -lt 2 ] && ViashError Not enough arguments passed to --trusted_sources. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --trusted_sources=*) + if [ -z "$VIASH_PAR_TRUSTED_SOURCES" ]; then + VIASH_PAR_TRUSTED_SOURCES=$(ViashRemoveFlags "$1") + else + VIASH_PAR_TRUSTED_SOURCES="$VIASH_PAR_TRUSTED_SOURCES;"$(ViashRemoveFlags "$1") + fi + shift 1 + ;; + --transcript_to_gene_map) + [ -n "$VIASH_PAR_TRANSCRIPT_TO_GENE_MAP" ] && ViashError Bad arguments for option \'--transcript_to_gene_map\': \'$VIASH_PAR_TRANSCRIPT_TO_GENE_MAP\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_TRANSCRIPT_TO_GENE_MAP="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --transcript_to_gene_map. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --transcript_to_gene_map=*) + [ -n "$VIASH_PAR_TRANSCRIPT_TO_GENE_MAP" ] && ViashError Bad arguments for option \'--transcript_to_gene_map=*\': \'$VIASH_PAR_TRANSCRIPT_TO_GENE_MAP\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_TRANSCRIPT_TO_GENE_MAP=$(ViashRemoveFlags "$1") + shift 1 + ;; + --allele_to_gene_map) + [ -n "$VIASH_PAR_ALLELE_TO_GENE_MAP" ] && ViashError Bad arguments for option \'--allele_to_gene_map\': \'$VIASH_PAR_ALLELE_TO_GENE_MAP\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_ALLELE_TO_GENE_MAP="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --allele_to_gene_map. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --allele_to_gene_map=*) + [ -n "$VIASH_PAR_ALLELE_TO_GENE_MAP" ] && ViashError Bad arguments for option \'--allele_to_gene_map=*\': \'$VIASH_PAR_ALLELE_TO_GENE_MAP\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_ALLELE_TO_GENE_MAP=$(ViashRemoveFlags "$1") + shift 1 + ;; + --polyA) + [ -n "$VIASH_PAR_POLYA" ] && ViashError Bad arguments for option \'--polyA\': \'$VIASH_PAR_POLYA\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_POLYA=true + shift 1 + ;; + --polyA_length) + [ -n "$VIASH_PAR_POLYA_LENGTH" ] && ViashError Bad arguments for option \'--polyA_length\': \'$VIASH_PAR_POLYA_LENGTH\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_POLYA_LENGTH="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --polyA_length. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --polyA_length=*) + [ -n "$VIASH_PAR_POLYA_LENGTH" ] && ViashError Bad arguments for option \'--polyA_length=*\': \'$VIASH_PAR_POLYA_LENGTH\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_POLYA_LENGTH=$(ViashRemoveFlags "$1") + shift 1 + ;; + --no_polyA_subset) + [ -n "$VIASH_PAR_NO_POLYA_SUBSET" ] && ViashError Bad arguments for option \'--no_polyA_subset\': \'$VIASH_PAR_NO_POLYA_SUBSET\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_NO_POLYA_SUBSET="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --no_polyA_subset. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --no_polyA_subset=*) + [ -n "$VIASH_PAR_NO_POLYA_SUBSET" ] && ViashError Bad arguments for option \'--no_polyA_subset=*\': \'$VIASH_PAR_NO_POLYA_SUBSET\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_NO_POLYA_SUBSET=$(ViashRemoveFlags "$1") + shift 1 + ;; + --bowtie) + [ -n "$VIASH_PAR_BOWTIE" ] && ViashError Bad arguments for option \'--bowtie\': \'$VIASH_PAR_BOWTIE\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_BOWTIE=true + shift 1 + ;; + --bowtie2) + [ -n "$VIASH_PAR_BOWTIE2" ] && ViashError Bad arguments for option \'--bowtie2\': \'$VIASH_PAR_BOWTIE2\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_BOWTIE2=true + shift 1 + ;; + --star) + [ -n "$VIASH_PAR_STAR" ] && ViashError Bad arguments for option \'--star\': \'$VIASH_PAR_STAR\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_STAR=true + shift 1 + ;; + --star_sjdboverhang) + [ -n "$VIASH_PAR_STAR_SJDBOVERHANG" ] && ViashError Bad arguments for option \'--star_sjdboverhang\': \'$VIASH_PAR_STAR_SJDBOVERHANG\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_STAR_SJDBOVERHANG="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --star_sjdboverhang. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --star_sjdboverhang=*) + [ -n "$VIASH_PAR_STAR_SJDBOVERHANG" ] && ViashError Bad arguments for option \'--star_sjdboverhang=*\': \'$VIASH_PAR_STAR_SJDBOVERHANG\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_STAR_SJDBOVERHANG=$(ViashRemoveFlags "$1") + shift 1 + ;; + --hisat2_hca) + [ -n "$VIASH_PAR_HISAT2_HCA" ] && ViashError Bad arguments for option \'--hisat2_hca\': \'$VIASH_PAR_HISAT2_HCA\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_HISAT2_HCA=true + shift 1 + ;; + --quiet) + [ -n "$VIASH_PAR_QUIET" ] && ViashError Bad arguments for option \'--quiet\': \'$VIASH_PAR_QUIET\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_QUIET=true + shift 1 + ;; + -q) + [ -n "$VIASH_PAR_QUIET" ] && ViashError Bad arguments for option \'-q\': \'$VIASH_PAR_QUIET\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_QUIET=true + shift 1 + ;; + --prep_pRSEM) + [ -n "$VIASH_PAR_PREP_PRSEM" ] && ViashError Bad arguments for option \'--prep_pRSEM\': \'$VIASH_PAR_PREP_PRSEM\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_PREP_PRSEM=true + shift 1 + ;; + --mappability_bigwig_file) + [ -n "$VIASH_PAR_MAPPABILITY_BIGWIG_FILE" ] && ViashError Bad arguments for option \'--mappability_bigwig_file\': \'$VIASH_PAR_MAPPABILITY_BIGWIG_FILE\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_MAPPABILITY_BIGWIG_FILE="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --mappability_bigwig_file. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --mappability_bigwig_file=*) + [ -n "$VIASH_PAR_MAPPABILITY_BIGWIG_FILE" ] && ViashError Bad arguments for option \'--mappability_bigwig_file=*\': \'$VIASH_PAR_MAPPABILITY_BIGWIG_FILE\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_MAPPABILITY_BIGWIG_FILE=$(ViashRemoveFlags "$1") + shift 1 + ;; + ---engine) + VIASH_ENGINE_ID="$2" + shift 2 + ;; + ---engine=*) + VIASH_ENGINE_ID="$(ViashRemoveFlags "$1")" + shift 1 + ;; + ---setup) + VIASH_MODE='setup' + VIASH_SETUP_STRATEGY="$2" + shift 2 + ;; + ---setup=*) + VIASH_MODE='setup' + VIASH_SETUP_STRATEGY="$(ViashRemoveFlags "$1")" + shift 1 + ;; + ---dockerfile) + VIASH_MODE='dockerfile' + shift 1 + ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; + ---debug) + VIASH_MODE='debug' + shift 1 + ;; + ---cpus) + [ -n "$VIASH_META_CPUS" ] && ViashError Bad arguments for option \'---cpus\': \'$VIASH_META_CPUS\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_META_CPUS="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to ---cpus. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + ---cpus=*) + [ -n "$VIASH_META_CPUS" ] && ViashError Bad arguments for option \'---cpus=*\': \'$VIASH_META_CPUS\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_META_CPUS=$(ViashRemoveFlags "$1") + shift 1 + ;; + ---memory) + [ -n "$VIASH_META_MEMORY" ] && ViashError Bad arguments for option \'---memory\': \'$VIASH_META_MEMORY\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_META_MEMORY="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to ---memory. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + ---memory=*) + [ -n "$VIASH_META_MEMORY" ] && ViashError Bad arguments for option \'---memory=*\': \'$VIASH_META_MEMORY\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_META_MEMORY=$(ViashRemoveFlags "$1") + shift 1 + ;; + *) # positional arg or unknown option + # since the positional args will be eval'd, can we always quote, instead of using ViashQuote + VIASH_POSITIONAL_ARGS="$VIASH_POSITIONAL_ARGS '$1'" + [[ $1 == -* ]] && ViashWarning $1 looks like a parameter but is not a defined parameter and will instead be treated as a positional argument. Use "--help" to get more information on the parameters. + shift # past argument + ;; + esac +done + +# parse positional parameters +eval set -- $VIASH_POSITIONAL_ARGS + + +if [ "$VIASH_ENGINE_ID" == "native" ] ; then + VIASH_ENGINE_TYPE='native' +elif [ "$VIASH_ENGINE_ID" == "docker" ] ; then + VIASH_ENGINE_TYPE='docker' +else + ViashError "Engine '$VIASH_ENGINE_ID' is not recognized. Options are: docker, native." + exit 1 +fi + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + # check if docker is installed properly + ViashDockerInstallationCheck + + # determine docker image id + if [[ "$VIASH_ENGINE_ID" == 'docker' ]]; then + VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/biobox/rsem/rsem_prepare_reference:main' + fi + + # print dockerfile + if [ "$VIASH_MODE" == "dockerfile" ]; then + ViashDockerfile "$VIASH_ENGINE_ID" + exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 + + # enter docker container + elif [[ "$VIASH_MODE" == "debug" ]]; then + VIASH_CMD="docker run --entrypoint=bash ${VIASH_DOCKER_RUN_ARGS[@]} -v '$(pwd)':/pwd --workdir /pwd -t $VIASH_DOCKER_IMAGE_ID" + ViashNotice "+ $VIASH_CMD" + eval $VIASH_CMD + exit + + # build docker image + elif [ "$VIASH_MODE" == "setup" ]; then + ViashDockerSetup "$VIASH_DOCKER_IMAGE_ID" "$VIASH_SETUP_STRATEGY" + ViashDockerCheckCommands "$VIASH_DOCKER_IMAGE_ID" 'ps' 'bash' + exit 0 + fi + + # check if docker image exists + ViashDockerSetup "$VIASH_DOCKER_IMAGE_ID" ifneedbepullelsecachedbuild + ViashDockerCheckCommands "$VIASH_DOCKER_IMAGE_ID" 'ps' 'bash' +fi + +# setting computational defaults + +# helper function for parsing memory strings +function ViashMemoryAsBytes { + local memory=`echo "$1" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]'` + local memory_regex='^([0-9]+)([kmgtp]i?b?|b)$' + if [[ $memory =~ $memory_regex ]]; then + local number=${memory/[^0-9]*/} + local symbol=${memory/*[0-9]/} + + case $symbol in + b) memory_b=$number ;; + kb|k) memory_b=$(( $number * 1000 )) ;; + mb|m) memory_b=$(( $number * 1000 * 1000 )) ;; + gb|g) memory_b=$(( $number * 1000 * 1000 * 1000 )) ;; + tb|t) memory_b=$(( $number * 1000 * 1000 * 1000 * 1000 )) ;; + pb|p) memory_b=$(( $number * 1000 * 1000 * 1000 * 1000 * 1000 )) ;; + kib|ki) memory_b=$(( $number * 1024 )) ;; + mib|mi) memory_b=$(( $number * 1024 * 1024 )) ;; + gib|gi) memory_b=$(( $number * 1024 * 1024 * 1024 )) ;; + tib|ti) memory_b=$(( $number * 1024 * 1024 * 1024 * 1024 )) ;; + pib|pi) memory_b=$(( $number * 1024 * 1024 * 1024 * 1024 * 1024 )) ;; + esac + echo "$memory_b" + fi +} +# compute memory in different units +if [ ! -z ${VIASH_META_MEMORY+x} ]; then + VIASH_META_MEMORY_B=`ViashMemoryAsBytes $VIASH_META_MEMORY` + # do not define other variables if memory_b is an empty string + if [ ! -z "$VIASH_META_MEMORY_B" ]; then + VIASH_META_MEMORY_KB=$(( ($VIASH_META_MEMORY_B+999) / 1000 )) + VIASH_META_MEMORY_MB=$(( ($VIASH_META_MEMORY_KB+999) / 1000 )) + VIASH_META_MEMORY_GB=$(( ($VIASH_META_MEMORY_MB+999) / 1000 )) + VIASH_META_MEMORY_TB=$(( ($VIASH_META_MEMORY_GB+999) / 1000 )) + VIASH_META_MEMORY_PB=$(( ($VIASH_META_MEMORY_TB+999) / 1000 )) + VIASH_META_MEMORY_KIB=$(( ($VIASH_META_MEMORY_B+1023) / 1024 )) + VIASH_META_MEMORY_MIB=$(( ($VIASH_META_MEMORY_KIB+1023) / 1024 )) + VIASH_META_MEMORY_GIB=$(( ($VIASH_META_MEMORY_MIB+1023) / 1024 )) + VIASH_META_MEMORY_TIB=$(( ($VIASH_META_MEMORY_GIB+1023) / 1024 )) + VIASH_META_MEMORY_PIB=$(( ($VIASH_META_MEMORY_TIB+1023) / 1024 )) + else + # unset memory if string is empty + unset $VIASH_META_MEMORY_B + fi +fi +# unset nproc if string is empty +if [ -z "$VIASH_META_CPUS" ]; then + unset $VIASH_META_CPUS +fi + + +# check whether required parameters exist +if [ -z ${VIASH_PAR_REFERENCE_FASTA_FILES+x} ]; then + ViashError '--reference_fasta_files' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_PAR_REFERENCE_NAME+x} ]; then + ViashError '--reference_name' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_PAR_OUTPUT+x} ]; then + ViashError '--output' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_NAME+x} ]; then + ViashError 'name' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_FUNCTIONALITY_NAME+x} ]; then + ViashError 'functionality_name' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_RESOURCES_DIR+x} ]; then + ViashError 'resources_dir' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_EXECUTABLE+x} ]; then + ViashError 'executable' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_CONFIG+x} ]; then + ViashError 'config' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_TEMP_DIR+x} ]; then + ViashError 'temp_dir' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi + +# filling in defaults +if [ -z ${VIASH_PAR_GFF3_GENES_AS_TRANSCRIPTS+x} ]; then + VIASH_PAR_GFF3_GENES_AS_TRANSCRIPTS="false" +fi +if [ -z ${VIASH_PAR_POLYA+x} ]; then + VIASH_PAR_POLYA="false" +fi +if [ -z ${VIASH_PAR_BOWTIE+x} ]; then + VIASH_PAR_BOWTIE="false" +fi +if [ -z ${VIASH_PAR_BOWTIE2+x} ]; then + VIASH_PAR_BOWTIE2="false" +fi +if [ -z ${VIASH_PAR_STAR+x} ]; then + VIASH_PAR_STAR="false" +fi +if [ -z ${VIASH_PAR_HISAT2_HCA+x} ]; then + VIASH_PAR_HISAT2_HCA="false" +fi +if [ -z ${VIASH_PAR_QUIET+x} ]; then + VIASH_PAR_QUIET="false" +fi +if [ -z ${VIASH_PAR_PREP_PRSEM+x} ]; then + VIASH_PAR_PREP_PRSEM="false" +fi + +# check whether required files exist +if [ ! -z "$VIASH_PAR_REFERENCE_FASTA_FILES" ]; then + IFS=';' + set -f + for file in $VIASH_PAR_REFERENCE_FASTA_FILES; do + unset IFS + if [ ! -e "$file" ]; then + ViashError "Input file '$file' does not exist." + exit 1 + fi + done + set +f +fi +if [ ! -z "$VIASH_PAR_GTF" ] && [ ! -e "$VIASH_PAR_GTF" ]; then + ViashError "Input file '$VIASH_PAR_GTF' does not exist." + exit 1 +fi +if [ ! -z "$VIASH_PAR_GFF3" ] && [ ! -e "$VIASH_PAR_GFF3" ]; then + ViashError "Input file '$VIASH_PAR_GFF3' does not exist." + exit 1 +fi +if [ ! -z "$VIASH_PAR_TRANSCRIPT_TO_GENE_MAP" ] && [ ! -e "$VIASH_PAR_TRANSCRIPT_TO_GENE_MAP" ]; then + ViashError "Input file '$VIASH_PAR_TRANSCRIPT_TO_GENE_MAP' does not exist." + exit 1 +fi +if [ ! -z "$VIASH_PAR_ALLELE_TO_GENE_MAP" ] && [ ! -e "$VIASH_PAR_ALLELE_TO_GENE_MAP" ]; then + ViashError "Input file '$VIASH_PAR_ALLELE_TO_GENE_MAP' does not exist." + exit 1 +fi +if [ ! -z "$VIASH_PAR_NO_POLYA_SUBSET" ] && [ ! -e "$VIASH_PAR_NO_POLYA_SUBSET" ]; then + ViashError "Input file '$VIASH_PAR_NO_POLYA_SUBSET' does not exist." + exit 1 +fi +if [ ! -z "$VIASH_PAR_MAPPABILITY_BIGWIG_FILE" ] && [ ! -e "$VIASH_PAR_MAPPABILITY_BIGWIG_FILE" ]; then + ViashError "Input file '$VIASH_PAR_MAPPABILITY_BIGWIG_FILE' does not exist." + exit 1 +fi + +# check whether parameters values are of the right type +if [[ -n "$VIASH_PAR_GFF3_GENES_AS_TRANSCRIPTS" ]]; then + if ! [[ "$VIASH_PAR_GFF3_GENES_AS_TRANSCRIPTS" =~ ^(true|True|TRUE|false|False|FALSE|yes|Yes|YES|no|No|NO)$ ]]; then + ViashError '--gff3_genes_as_transcripts' has to be a boolean_true. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_PAR_POLYA" ]]; then + if ! [[ "$VIASH_PAR_POLYA" =~ ^(true|True|TRUE|false|False|FALSE|yes|Yes|YES|no|No|NO)$ ]]; then + ViashError '--polyA' has to be a boolean_true. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_PAR_POLYA_LENGTH" ]]; then + if ! [[ "$VIASH_PAR_POLYA_LENGTH" =~ ^[-+]?[0-9]+$ ]]; then + ViashError '--polyA_length' has to be an integer. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_PAR_BOWTIE" ]]; then + if ! [[ "$VIASH_PAR_BOWTIE" =~ ^(true|True|TRUE|false|False|FALSE|yes|Yes|YES|no|No|NO)$ ]]; then + ViashError '--bowtie' has to be a boolean_true. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_PAR_BOWTIE2" ]]; then + if ! [[ "$VIASH_PAR_BOWTIE2" =~ ^(true|True|TRUE|false|False|FALSE|yes|Yes|YES|no|No|NO)$ ]]; then + ViashError '--bowtie2' has to be a boolean_true. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_PAR_STAR" ]]; then + if ! [[ "$VIASH_PAR_STAR" =~ ^(true|True|TRUE|false|False|FALSE|yes|Yes|YES|no|No|NO)$ ]]; then + ViashError '--star' has to be a boolean_true. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_PAR_STAR_SJDBOVERHANG" ]]; then + if ! [[ "$VIASH_PAR_STAR_SJDBOVERHANG" =~ ^[-+]?[0-9]+$ ]]; then + ViashError '--star_sjdboverhang' has to be an integer. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_PAR_HISAT2_HCA" ]]; then + if ! [[ "$VIASH_PAR_HISAT2_HCA" =~ ^(true|True|TRUE|false|False|FALSE|yes|Yes|YES|no|No|NO)$ ]]; then + ViashError '--hisat2_hca' has to be a boolean_true. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_PAR_QUIET" ]]; then + if ! [[ "$VIASH_PAR_QUIET" =~ ^(true|True|TRUE|false|False|FALSE|yes|Yes|YES|no|No|NO)$ ]]; then + ViashError '--quiet' has to be a boolean_true. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_PAR_PREP_PRSEM" ]]; then + if ! [[ "$VIASH_PAR_PREP_PRSEM" =~ ^(true|True|TRUE|false|False|FALSE|yes|Yes|YES|no|No|NO)$ ]]; then + ViashError '--prep_pRSEM' has to be a boolean_true. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_CPUS" ]]; then + if ! [[ "$VIASH_META_CPUS" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'cpus' has to be an integer. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_B" ]]; then + if ! [[ "$VIASH_META_MEMORY_B" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_b' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_KB" ]]; then + if ! [[ "$VIASH_META_MEMORY_KB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_kb' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_MB" ]]; then + if ! [[ "$VIASH_META_MEMORY_MB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_mb' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_GB" ]]; then + if ! [[ "$VIASH_META_MEMORY_GB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_gb' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_TB" ]]; then + if ! [[ "$VIASH_META_MEMORY_TB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_tb' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_PB" ]]; then + if ! [[ "$VIASH_META_MEMORY_PB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_pb' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_KIB" ]]; then + if ! [[ "$VIASH_META_MEMORY_KIB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_kib' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_MIB" ]]; then + if ! [[ "$VIASH_META_MEMORY_MIB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_mib' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_GIB" ]]; then + if ! [[ "$VIASH_META_MEMORY_GIB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_gib' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_TIB" ]]; then + if ! [[ "$VIASH_META_MEMORY_TIB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_tib' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_PIB" ]]; then + if ! [[ "$VIASH_META_MEMORY_PIB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_pib' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi + +# create parent directories of output files, if so desired +if [ ! -z "$VIASH_PAR_OUTPUT" ] && [ ! -d "$(dirname "$VIASH_PAR_OUTPUT")" ]; then + mkdir -p "$(dirname "$VIASH_PAR_OUTPUT")" +fi + +if [ "$VIASH_ENGINE_ID" == "native" ] ; then + if [ "$VIASH_MODE" == "run" ]; then + VIASH_CMD="bash" + else + ViashError "Engine '$VIASH_ENGINE_ID' does not support mode '$VIASH_MODE'." + exit 1 + fi +fi + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + # detect volumes from file arguments + VIASH_CHOWN_VARS=() +if [ ! -z "$VIASH_PAR_REFERENCE_FASTA_FILES" ]; then + VIASH_TEST_REFERENCE_FASTA_FILES=() + IFS=';' + for var in $VIASH_PAR_REFERENCE_FASTA_FILES; do + unset IFS + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$var")" ) + var=$(ViashDockerAutodetectMount "$var") + VIASH_TEST_REFERENCE_FASTA_FILES+=( "$var" ) + done + VIASH_PAR_REFERENCE_FASTA_FILES=$(IFS=';' ; echo "${VIASH_TEST_REFERENCE_FASTA_FILES[*]}") +fi +if [ ! -z "$VIASH_PAR_OUTPUT" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_PAR_OUTPUT")" ) + VIASH_PAR_OUTPUT=$(ViashDockerAutodetectMount "$VIASH_PAR_OUTPUT") + VIASH_CHOWN_VARS+=( "$VIASH_PAR_OUTPUT" ) +fi +if [ ! -z "$VIASH_PAR_GTF" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_PAR_GTF")" ) + VIASH_PAR_GTF=$(ViashDockerAutodetectMount "$VIASH_PAR_GTF") +fi +if [ ! -z "$VIASH_PAR_GFF3" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_PAR_GFF3")" ) + VIASH_PAR_GFF3=$(ViashDockerAutodetectMount "$VIASH_PAR_GFF3") +fi +if [ ! -z "$VIASH_PAR_TRANSCRIPT_TO_GENE_MAP" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_PAR_TRANSCRIPT_TO_GENE_MAP")" ) + VIASH_PAR_TRANSCRIPT_TO_GENE_MAP=$(ViashDockerAutodetectMount "$VIASH_PAR_TRANSCRIPT_TO_GENE_MAP") +fi +if [ ! -z "$VIASH_PAR_ALLELE_TO_GENE_MAP" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_PAR_ALLELE_TO_GENE_MAP")" ) + VIASH_PAR_ALLELE_TO_GENE_MAP=$(ViashDockerAutodetectMount "$VIASH_PAR_ALLELE_TO_GENE_MAP") +fi +if [ ! -z "$VIASH_PAR_NO_POLYA_SUBSET" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_PAR_NO_POLYA_SUBSET")" ) + VIASH_PAR_NO_POLYA_SUBSET=$(ViashDockerAutodetectMount "$VIASH_PAR_NO_POLYA_SUBSET") +fi +if [ ! -z "$VIASH_PAR_MAPPABILITY_BIGWIG_FILE" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_PAR_MAPPABILITY_BIGWIG_FILE")" ) + VIASH_PAR_MAPPABILITY_BIGWIG_FILE=$(ViashDockerAutodetectMount "$VIASH_PAR_MAPPABILITY_BIGWIG_FILE") +fi +if [ ! -z "$VIASH_META_RESOURCES_DIR" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_RESOURCES_DIR")" ) + VIASH_META_RESOURCES_DIR=$(ViashDockerAutodetectMount "$VIASH_META_RESOURCES_DIR") +fi +if [ ! -z "$VIASH_META_EXECUTABLE" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_EXECUTABLE")" ) + VIASH_META_EXECUTABLE=$(ViashDockerAutodetectMount "$VIASH_META_EXECUTABLE") +fi +if [ ! -z "$VIASH_META_CONFIG" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_CONFIG")" ) + VIASH_META_CONFIG=$(ViashDockerAutodetectMount "$VIASH_META_CONFIG") +fi +if [ ! -z "$VIASH_META_TEMP_DIR" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_TEMP_DIR")" ) + VIASH_META_TEMP_DIR=$(ViashDockerAutodetectMount "$VIASH_META_TEMP_DIR") +fi + + # get unique mounts + VIASH_UNIQUE_MOUNTS=($(for val in "${VIASH_DIRECTORY_MOUNTS[@]}"; do echo "$val"; done | sort -u)) +fi + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + # change file ownership + function ViashPerformChown { + if (( ${#VIASH_CHOWN_VARS[@]} )); then + set +e + VIASH_CMD="docker run --entrypoint=bash --rm ${VIASH_UNIQUE_MOUNTS[@]} $VIASH_DOCKER_IMAGE_ID -c 'chown $(id -u):$(id -g) --silent --recursive ${VIASH_CHOWN_VARS[@]}'" + ViashDebug "+ $VIASH_CMD" + eval $VIASH_CMD + set -e + fi + } + trap ViashPerformChown EXIT +fi + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + # helper function for filling in extra docker args + if [ ! -z "$VIASH_META_MEMORY_B" ]; then + VIASH_DOCKER_RUN_ARGS+=("--memory=${VIASH_META_MEMORY_B}") + fi + if [ ! -z "$VIASH_META_CPUS" ]; then + VIASH_DOCKER_RUN_ARGS+=("--cpus=${VIASH_META_CPUS}") + fi +fi + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + VIASH_CMD="docker run --entrypoint=bash ${VIASH_DOCKER_RUN_ARGS[@]} ${VIASH_UNIQUE_MOUNTS[@]} $VIASH_DOCKER_IMAGE_ID" +fi + + +# set dependency paths + + +ViashDebug "Running command: $(echo $VIASH_CMD)" +cat << VIASHEOF | eval $VIASH_CMD +set -e +tempscript=\$(mktemp "$VIASH_META_TEMP_DIR/viash-run-rsem_prepare_reference-XXXXXX").sh +function clean_up { + rm "\$tempscript" +} +function interrupt { + echo -e "\nCTRL-C Pressed..." + exit 1 +} +trap clean_up EXIT +trap interrupt INT SIGINT +cat > "\$tempscript" << 'VIASHMAIN' +## VIASH START +# The following code has been auto-generated by Viash. +$( if [ ! -z ${VIASH_PAR_REFERENCE_FASTA_FILES+x} ]; then echo "${VIASH_PAR_REFERENCE_FASTA_FILES}" | sed "s#'#'\"'\"'#g;s#.*#par_reference_fasta_files='&'#" ; else echo "# par_reference_fasta_files="; fi ) +$( if [ ! -z ${VIASH_PAR_REFERENCE_NAME+x} ]; then echo "${VIASH_PAR_REFERENCE_NAME}" | sed "s#'#'\"'\"'#g;s#.*#par_reference_name='&'#" ; else echo "# par_reference_name="; fi ) +$( if [ ! -z ${VIASH_PAR_OUTPUT+x} ]; then echo "${VIASH_PAR_OUTPUT}" | sed "s#'#'\"'\"'#g;s#.*#par_output='&'#" ; else echo "# par_output="; fi ) +$( if [ ! -z ${VIASH_PAR_GTF+x} ]; then echo "${VIASH_PAR_GTF}" | sed "s#'#'\"'\"'#g;s#.*#par_gtf='&'#" ; else echo "# par_gtf="; fi ) +$( if [ ! -z ${VIASH_PAR_GFF3+x} ]; then echo "${VIASH_PAR_GFF3}" | sed "s#'#'\"'\"'#g;s#.*#par_gff3='&'#" ; else echo "# par_gff3="; fi ) +$( if [ ! -z ${VIASH_PAR_GFF3_RNA_PATTERNS+x} ]; then echo "${VIASH_PAR_GFF3_RNA_PATTERNS}" | sed "s#'#'\"'\"'#g;s#.*#par_gff3_rna_patterns='&'#" ; else echo "# par_gff3_rna_patterns="; fi ) +$( if [ ! -z ${VIASH_PAR_GFF3_GENES_AS_TRANSCRIPTS+x} ]; then echo "${VIASH_PAR_GFF3_GENES_AS_TRANSCRIPTS}" | sed "s#'#'\"'\"'#g;s#.*#par_gff3_genes_as_transcripts='&'#" ; else echo "# par_gff3_genes_as_transcripts="; fi ) +$( if [ ! -z ${VIASH_PAR_TRUSTED_SOURCES+x} ]; then echo "${VIASH_PAR_TRUSTED_SOURCES}" | sed "s#'#'\"'\"'#g;s#.*#par_trusted_sources='&'#" ; else echo "# par_trusted_sources="; fi ) +$( if [ ! -z ${VIASH_PAR_TRANSCRIPT_TO_GENE_MAP+x} ]; then echo "${VIASH_PAR_TRANSCRIPT_TO_GENE_MAP}" | sed "s#'#'\"'\"'#g;s#.*#par_transcript_to_gene_map='&'#" ; else echo "# par_transcript_to_gene_map="; fi ) +$( if [ ! -z ${VIASH_PAR_ALLELE_TO_GENE_MAP+x} ]; then echo "${VIASH_PAR_ALLELE_TO_GENE_MAP}" | sed "s#'#'\"'\"'#g;s#.*#par_allele_to_gene_map='&'#" ; else echo "# par_allele_to_gene_map="; fi ) +$( if [ ! -z ${VIASH_PAR_POLYA+x} ]; then echo "${VIASH_PAR_POLYA}" | sed "s#'#'\"'\"'#g;s#.*#par_polyA='&'#" ; else echo "# par_polyA="; fi ) +$( if [ ! -z ${VIASH_PAR_POLYA_LENGTH+x} ]; then echo "${VIASH_PAR_POLYA_LENGTH}" | sed "s#'#'\"'\"'#g;s#.*#par_polyA_length='&'#" ; else echo "# par_polyA_length="; fi ) +$( if [ ! -z ${VIASH_PAR_NO_POLYA_SUBSET+x} ]; then echo "${VIASH_PAR_NO_POLYA_SUBSET}" | sed "s#'#'\"'\"'#g;s#.*#par_no_polyA_subset='&'#" ; else echo "# par_no_polyA_subset="; fi ) +$( if [ ! -z ${VIASH_PAR_BOWTIE+x} ]; then echo "${VIASH_PAR_BOWTIE}" | sed "s#'#'\"'\"'#g;s#.*#par_bowtie='&'#" ; else echo "# par_bowtie="; fi ) +$( if [ ! -z ${VIASH_PAR_BOWTIE2+x} ]; then echo "${VIASH_PAR_BOWTIE2}" | sed "s#'#'\"'\"'#g;s#.*#par_bowtie2='&'#" ; else echo "# par_bowtie2="; fi ) +$( if [ ! -z ${VIASH_PAR_STAR+x} ]; then echo "${VIASH_PAR_STAR}" | sed "s#'#'\"'\"'#g;s#.*#par_star='&'#" ; else echo "# par_star="; fi ) +$( if [ ! -z ${VIASH_PAR_STAR_SJDBOVERHANG+x} ]; then echo "${VIASH_PAR_STAR_SJDBOVERHANG}" | sed "s#'#'\"'\"'#g;s#.*#par_star_sjdboverhang='&'#" ; else echo "# par_star_sjdboverhang="; fi ) +$( if [ ! -z ${VIASH_PAR_HISAT2_HCA+x} ]; then echo "${VIASH_PAR_HISAT2_HCA}" | sed "s#'#'\"'\"'#g;s#.*#par_hisat2_hca='&'#" ; else echo "# par_hisat2_hca="; fi ) +$( if [ ! -z ${VIASH_PAR_QUIET+x} ]; then echo "${VIASH_PAR_QUIET}" | sed "s#'#'\"'\"'#g;s#.*#par_quiet='&'#" ; else echo "# par_quiet="; fi ) +$( if [ ! -z ${VIASH_PAR_PREP_PRSEM+x} ]; then echo "${VIASH_PAR_PREP_PRSEM}" | sed "s#'#'\"'\"'#g;s#.*#par_prep_pRSEM='&'#" ; else echo "# par_prep_pRSEM="; fi ) +$( if [ ! -z ${VIASH_PAR_MAPPABILITY_BIGWIG_FILE+x} ]; then echo "${VIASH_PAR_MAPPABILITY_BIGWIG_FILE}" | sed "s#'#'\"'\"'#g;s#.*#par_mappability_bigwig_file='&'#" ; else echo "# par_mappability_bigwig_file="; fi ) +$( if [ ! -z ${VIASH_META_NAME+x} ]; then echo "${VIASH_META_NAME}" | sed "s#'#'\"'\"'#g;s#.*#meta_name='&'#" ; else echo "# meta_name="; fi ) +$( if [ ! -z ${VIASH_META_FUNCTIONALITY_NAME+x} ]; then echo "${VIASH_META_FUNCTIONALITY_NAME}" | sed "s#'#'\"'\"'#g;s#.*#meta_functionality_name='&'#" ; else echo "# meta_functionality_name="; fi ) +$( if [ ! -z ${VIASH_META_RESOURCES_DIR+x} ]; then echo "${VIASH_META_RESOURCES_DIR}" | sed "s#'#'\"'\"'#g;s#.*#meta_resources_dir='&'#" ; else echo "# meta_resources_dir="; fi ) +$( if [ ! -z ${VIASH_META_EXECUTABLE+x} ]; then echo "${VIASH_META_EXECUTABLE}" | sed "s#'#'\"'\"'#g;s#.*#meta_executable='&'#" ; else echo "# meta_executable="; fi ) +$( if [ ! -z ${VIASH_META_CONFIG+x} ]; then echo "${VIASH_META_CONFIG}" | sed "s#'#'\"'\"'#g;s#.*#meta_config='&'#" ; else echo "# meta_config="; fi ) +$( if [ ! -z ${VIASH_META_TEMP_DIR+x} ]; then echo "${VIASH_META_TEMP_DIR}" | sed "s#'#'\"'\"'#g;s#.*#meta_temp_dir='&'#" ; else echo "# meta_temp_dir="; fi ) +$( if [ ! -z ${VIASH_META_CPUS+x} ]; then echo "${VIASH_META_CPUS}" | sed "s#'#'\"'\"'#g;s#.*#meta_cpus='&'#" ; else echo "# meta_cpus="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_B+x} ]; then echo "${VIASH_META_MEMORY_B}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_b='&'#" ; else echo "# meta_memory_b="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_KB+x} ]; then echo "${VIASH_META_MEMORY_KB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_kb='&'#" ; else echo "# meta_memory_kb="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_MB+x} ]; then echo "${VIASH_META_MEMORY_MB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_mb='&'#" ; else echo "# meta_memory_mb="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_GB+x} ]; then echo "${VIASH_META_MEMORY_GB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_gb='&'#" ; else echo "# meta_memory_gb="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_TB+x} ]; then echo "${VIASH_META_MEMORY_TB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_tb='&'#" ; else echo "# meta_memory_tb="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_PB+x} ]; then echo "${VIASH_META_MEMORY_PB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_pb='&'#" ; else echo "# meta_memory_pb="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_KIB+x} ]; then echo "${VIASH_META_MEMORY_KIB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_kib='&'#" ; else echo "# meta_memory_kib="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_MIB+x} ]; then echo "${VIASH_META_MEMORY_MIB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_mib='&'#" ; else echo "# meta_memory_mib="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_GIB+x} ]; then echo "${VIASH_META_MEMORY_GIB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_gib='&'#" ; else echo "# meta_memory_gib="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_TIB+x} ]; then echo "${VIASH_META_MEMORY_TIB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_tib='&'#" ; else echo "# meta_memory_tib="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_PIB+x} ]; then echo "${VIASH_META_MEMORY_PIB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_pib='&'#" ; else echo "# meta_memory_pib="; fi ) + +## VIASH END +#!/bin/bash + +set -eo pipefail + +unset_if_false=( par_gff3_genes_as_transcripts par_polyA par_bowtie par_bowtie2 par_star par_hisat2_hca par_quiet par_prep_pRSEM ) + +for par in \${unset_if_false[@]}; do + test_val="\${!par}" + [[ "\$test_val" == "false" ]] && unset \$par +done + +# replace ';' with ',' +par_reference_fasta_files=\$(echo \$par_reference_fasta_files | tr ';' ',') +par_gff3_rna_patterns=\$(echo \$par_gff3_rna_patterns | tr ';' ',') +par_trusted_sources=\$(echo \$par_trusted_sources | tr ';' ',') + +echo "\$par_reference_fasta_files" +rsem-prepare-reference \\ + \${par_gtf:+--gtf "\${par_gtf}"} \\ + \${par_gff3:+--gff3 "\${par_gff3}"} \\ + \${par_gff3_rna_patterns:+--gff3-RNA-patterns "\${par_gff3_rna_patterns}"} \\ + \${par_gff3_genes_as_transcripts:+--gff3-genes-as-transcripts "\${par_gff3_genes_as_transcripts}"} \\ + \${par_trusted_sources:+--trusted-sources "\${par_trusted_sources}"} \\ + \${par_transcript_to_gene_map:+--transcript-to-gene-map "\${par_transcript_to_gene_map}"} \\ + \${par_allele_to_gene_map:+--allele-to-gene-map "\${par_allele_to_gene_map}"} \\ + \${par_polyA:+--polyA} \\ + \${par_polyA_length:+--polyA-length "\${par_polyA_length}"} \\ + \${par_no_polyA_subset:+--no-polyA-subset "\${par_no_polyA_subset}"} \\ + \${par_bowtie:+--bowtie} \\ + \${par_bowtie2:+--bowtie2} \\ + \${par_star:+--star} \\ + \${par_star_sjdboverhang:+--star-sjdboverhang "\${par_star_sjdboverhang}"} \\ + \${par_hisat2_hca:+--hisat2-hca} \\ + \${par_quiet:+--quiet} \\ + \${par_prep_pRSEM:+--prep-pRSEM} \\ + \${par_mappability_bigwig_file:+--mappability-bigwig-file "\${par_mappability_bigwig_file}"} \\ + \${meta_cpus:+--num-threads "\${meta_cpus}"} \\ + "\${par_reference_fasta_files}" \\ + "\${par_reference_name}" + +mkdir -p "\${par_output}" +mv \${par_reference_name}.* "\${par_output}/" +VIASHMAIN +bash "\$tempscript" & +wait "\$!" + +VIASHEOF + + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + # strip viash automount from file paths + + if [ ! -z "$VIASH_PAR_REFERENCE_FASTA_FILES" ]; then + unset VIASH_TEST_REFERENCE_FASTA_FILES + IFS=';' + for var in $VIASH_PAR_REFERENCE_FASTA_FILES; do + unset IFS + if [ -z "$VIASH_TEST_REFERENCE_FASTA_FILES" ]; then + VIASH_TEST_REFERENCE_FASTA_FILES="$(ViashDockerStripAutomount "$var")" + else + VIASH_TEST_REFERENCE_FASTA_FILES="$VIASH_TEST_REFERENCE_FASTA_FILES;""$(ViashDockerStripAutomount "$var")" + fi + done + VIASH_PAR_REFERENCE_FASTA_FILES="$VIASH_TEST_REFERENCE_FASTA_FILES" + fi + if [ ! -z "$VIASH_PAR_OUTPUT" ]; then + VIASH_PAR_OUTPUT=$(ViashDockerStripAutomount "$VIASH_PAR_OUTPUT") + fi + if [ ! -z "$VIASH_PAR_GTF" ]; then + VIASH_PAR_GTF=$(ViashDockerStripAutomount "$VIASH_PAR_GTF") + fi + if [ ! -z "$VIASH_PAR_GFF3" ]; then + VIASH_PAR_GFF3=$(ViashDockerStripAutomount "$VIASH_PAR_GFF3") + fi + if [ ! -z "$VIASH_PAR_TRANSCRIPT_TO_GENE_MAP" ]; then + VIASH_PAR_TRANSCRIPT_TO_GENE_MAP=$(ViashDockerStripAutomount "$VIASH_PAR_TRANSCRIPT_TO_GENE_MAP") + fi + if [ ! -z "$VIASH_PAR_ALLELE_TO_GENE_MAP" ]; then + VIASH_PAR_ALLELE_TO_GENE_MAP=$(ViashDockerStripAutomount "$VIASH_PAR_ALLELE_TO_GENE_MAP") + fi + if [ ! -z "$VIASH_PAR_NO_POLYA_SUBSET" ]; then + VIASH_PAR_NO_POLYA_SUBSET=$(ViashDockerStripAutomount "$VIASH_PAR_NO_POLYA_SUBSET") + fi + if [ ! -z "$VIASH_PAR_MAPPABILITY_BIGWIG_FILE" ]; then + VIASH_PAR_MAPPABILITY_BIGWIG_FILE=$(ViashDockerStripAutomount "$VIASH_PAR_MAPPABILITY_BIGWIG_FILE") + fi + if [ ! -z "$VIASH_META_RESOURCES_DIR" ]; then + VIASH_META_RESOURCES_DIR=$(ViashDockerStripAutomount "$VIASH_META_RESOURCES_DIR") + fi + if [ ! -z "$VIASH_META_EXECUTABLE" ]; then + VIASH_META_EXECUTABLE=$(ViashDockerStripAutomount "$VIASH_META_EXECUTABLE") + fi + if [ ! -z "$VIASH_META_CONFIG" ]; then + VIASH_META_CONFIG=$(ViashDockerStripAutomount "$VIASH_META_CONFIG") + fi + if [ ! -z "$VIASH_META_TEMP_DIR" ]; then + VIASH_META_TEMP_DIR=$(ViashDockerStripAutomount "$VIASH_META_TEMP_DIR") + fi +fi + + +# check whether required files exist +if [ ! -z "$VIASH_PAR_OUTPUT" ] && [ ! -e "$VIASH_PAR_OUTPUT" ]; then + ViashError "Output file '$VIASH_PAR_OUTPUT' does not exist." + exit 1 +fi + + +exit 0 diff --git a/target/executable/salmon/salmon_index/.config.vsh.yaml b/target/executable/salmon/salmon_index/.config.vsh.yaml index e5d4f46f..6a5b2ba5 100644 --- a/target/executable/salmon/salmon_index/.config.vsh.yaml +++ b/target/executable/salmon/salmon_index/.config.vsh.yaml @@ -276,15 +276,15 @@ build_info: engine: "docker|native" output: "target/executable/salmon/salmon_index" executable: "target/executable/salmon/salmon_index/salmon_index" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/salmon/salmon_index/salmon_index b/target/executable/salmon/salmon_index/salmon_index index bc1f2ba0..021c812b 100755 --- a/target/executable/salmon/salmon_index/salmon_index +++ b/target/executable/salmon/salmon_index/salmon_index @@ -2,7 +2,7 @@ # salmon_index main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -57,24 +57,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -286,9 +286,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -340,9 +340,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -362,7 +362,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -386,9 +387,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -545,9 +546,9 @@ RUN salmon index -v 2>&1 | sed 's/salmon \([0-9.]*\)/salmon: \1/' > /var/softwar LABEL org.opencontainers.image.authors="Sai Nirmayi Yasa" LABEL org.opencontainers.image.description="Companion container for running component salmon salmon_index" -LABEL org.opencontainers.image.created="2024-08-21T11:36:03Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:51Z" LABEL org.opencontainers.image.source="https://github.com/COMBINE-lab/salmon" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -574,6 +575,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -604,14 +608,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -619,7 +626,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -627,7 +634,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -635,17 +644,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -839,6 +853,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -900,6 +926,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/salmon/salmon_quant/.config.vsh.yaml b/target/executable/salmon/salmon_quant/.config.vsh.yaml index f185c964..d241ab2d 100644 --- a/target/executable/salmon/salmon_quant/.config.vsh.yaml +++ b/target/executable/salmon/salmon_quant/.config.vsh.yaml @@ -1172,15 +1172,15 @@ build_info: engine: "docker|native" output: "target/executable/salmon/salmon_quant" executable: "target/executable/salmon/salmon_quant/salmon_quant" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/salmon/salmon_quant/salmon_quant b/target/executable/salmon/salmon_quant/salmon_quant index 0af7704a..3b3634a4 100755 --- a/target/executable/salmon/salmon_quant/salmon_quant +++ b/target/executable/salmon/salmon_quant/salmon_quant @@ -2,7 +2,7 @@ # salmon_quant main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -57,24 +57,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -908,9 +908,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -962,9 +962,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -984,7 +984,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -1008,9 +1009,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -1167,9 +1168,9 @@ RUN salmon index -v 2>&1 | sed 's/salmon \([0-9.]*\)/salmon: \1/' > /var/softwar LABEL org.opencontainers.image.authors="Sai Nirmayi Yasa" LABEL org.opencontainers.image.description="Companion container for running component salmon salmon_quant" -LABEL org.opencontainers.image.created="2024-08-21T11:36:03Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:52Z" LABEL org.opencontainers.image.source="https://github.com/COMBINE-lab/salmon" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -1196,6 +1197,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -1226,14 +1230,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -1241,7 +1248,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -1249,7 +1256,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -1257,17 +1266,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -2194,6 +2208,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -2255,6 +2281,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/samtools/samtools_collate/.config.vsh.yaml b/target/executable/samtools/samtools_collate/.config.vsh.yaml index 8d9dcd3b..29015d41 100644 --- a/target/executable/samtools/samtools_collate/.config.vsh.yaml +++ b/target/executable/samtools/samtools_collate/.config.vsh.yaml @@ -263,15 +263,15 @@ build_info: engine: "docker|native" output: "target/executable/samtools/samtools_collate" executable: "target/executable/samtools/samtools_collate/samtools_collate" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/samtools/samtools_collate/samtools_collate b/target/executable/samtools/samtools_collate/samtools_collate index 0da2dc4c..434814ed 100755 --- a/target/executable/samtools/samtools_collate/samtools_collate +++ b/target/executable/samtools/samtools_collate/samtools_collate @@ -2,7 +2,7 @@ # samtools_collate main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -57,24 +57,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -258,9 +258,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -312,9 +312,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -334,7 +334,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -358,9 +359,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -518,9 +519,9 @@ sed 's#Using ##;s# \([0-9\.]*\)$#: \1#' > /var/software_versions.txt LABEL org.opencontainers.image.authors="Emma Rousseau" LABEL org.opencontainers.image.description="Companion container for running component samtools samtools_collate" -LABEL org.opencontainers.image.created="2024-08-21T11:35:54Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:42Z" LABEL org.opencontainers.image.source="https://github.com/samtools/samtools" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -547,6 +548,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -577,14 +581,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -592,7 +599,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -600,7 +607,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -608,17 +617,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -834,6 +848,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -895,6 +921,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/samtools/samtools_faidx/.config.vsh.yaml b/target/executable/samtools/samtools_faidx/.config.vsh.yaml index 06c00810..82d37390 100644 --- a/target/executable/samtools/samtools_faidx/.config.vsh.yaml +++ b/target/executable/samtools/samtools_faidx/.config.vsh.yaml @@ -242,15 +242,15 @@ build_info: engine: "docker|native" output: "target/executable/samtools/samtools_faidx" executable: "target/executable/samtools/samtools_faidx/samtools_faidx" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/samtools/samtools_faidx/samtools_faidx b/target/executable/samtools/samtools_faidx/samtools_faidx index 5de56cf9..63ddeecb 100755 --- a/target/executable/samtools/samtools_faidx/samtools_faidx +++ b/target/executable/samtools/samtools_faidx/samtools_faidx @@ -2,7 +2,7 @@ # samtools_faidx main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -57,24 +57,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -251,9 +251,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -305,9 +305,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -327,7 +327,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -351,9 +352,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -511,9 +512,9 @@ sed 's#Using ##;s# \([0-9\.]*\)$#: \1#' > /var/software_versions.txt LABEL org.opencontainers.image.authors="Emma Rousseau" LABEL org.opencontainers.image.description="Companion container for running component samtools samtools_faidx" -LABEL org.opencontainers.image.created="2024-08-21T11:35:54Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:42Z" LABEL org.opencontainers.image.source="https://github.com/samtools/samtools" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -540,6 +541,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -570,14 +574,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -585,7 +592,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -593,7 +600,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -601,17 +610,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -777,6 +791,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -838,6 +864,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/samtools/samtools_fasta/.config.vsh.yaml b/target/executable/samtools/samtools_fasta/.config.vsh.yaml index e3107328..1f8dde3e 100644 --- a/target/executable/samtools/samtools_fasta/.config.vsh.yaml +++ b/target/executable/samtools/samtools_fasta/.config.vsh.yaml @@ -432,15 +432,15 @@ build_info: engine: "docker|native" output: "target/executable/samtools/samtools_fasta" executable: "target/executable/samtools/samtools_fasta/samtools_fasta" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/samtools/samtools_fasta/samtools_fasta b/target/executable/samtools/samtools_fasta/samtools_fasta index c4936296..46bf9ba4 100755 --- a/target/executable/samtools/samtools_fasta/samtools_fasta +++ b/target/executable/samtools/samtools_fasta/samtools_fasta @@ -2,7 +2,7 @@ # samtools_fasta main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -57,24 +57,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -364,9 +364,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -418,9 +418,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -440,7 +440,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -464,9 +465,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -624,9 +625,9 @@ sed 's#Using ##;s# \([0-9\.]*\)$#: \1#' > /var/software_versions.txt LABEL org.opencontainers.image.authors="Emma Rousseau" LABEL org.opencontainers.image.description="Companion container for running component samtools samtools_fasta" -LABEL org.opencontainers.image.created="2024-08-21T11:35:54Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:43Z" LABEL org.opencontainers.image.source="https://github.com/samtools/samtools" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -653,6 +654,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -683,14 +687,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -698,7 +705,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -706,7 +713,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -714,17 +723,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -1135,6 +1149,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -1196,6 +1222,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/samtools/samtools_fastq/.config.vsh.yaml b/target/executable/samtools/samtools_fastq/.config.vsh.yaml index 9d149fd0..8494263d 100644 --- a/target/executable/samtools/samtools_fastq/.config.vsh.yaml +++ b/target/executable/samtools/samtools_fastq/.config.vsh.yaml @@ -432,15 +432,15 @@ build_info: engine: "docker|native" output: "target/executable/samtools/samtools_fastq" executable: "target/executable/samtools/samtools_fastq/samtools_fastq" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/samtools/samtools_fastq/samtools_fastq b/target/executable/samtools/samtools_fastq/samtools_fastq index 183a1897..7540a4e1 100755 --- a/target/executable/samtools/samtools_fastq/samtools_fastq +++ b/target/executable/samtools/samtools_fastq/samtools_fastq @@ -2,7 +2,7 @@ # samtools_fastq main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -57,24 +57,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -365,9 +365,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -419,9 +419,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -441,7 +441,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -465,9 +466,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -625,9 +626,9 @@ sed 's#Using ##;s# \([0-9\.]*\)$#: \1#' > /var/software_versions.txt LABEL org.opencontainers.image.authors="Emma Rousseau" LABEL org.opencontainers.image.description="Companion container for running component samtools samtools_fastq" -LABEL org.opencontainers.image.created="2024-08-21T11:35:53Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:42Z" LABEL org.opencontainers.image.source="https://github.com/samtools/samtools" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -654,6 +655,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -684,14 +688,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -699,7 +706,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -707,7 +714,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -715,17 +724,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -1136,6 +1150,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -1197,6 +1223,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/samtools/samtools_flagstat/.config.vsh.yaml b/target/executable/samtools/samtools_flagstat/.config.vsh.yaml index 31cdef96..5feee031 100644 --- a/target/executable/samtools/samtools_flagstat/.config.vsh.yaml +++ b/target/executable/samtools/samtools_flagstat/.config.vsh.yaml @@ -172,15 +172,15 @@ build_info: engine: "docker|native" output: "target/executable/samtools/samtools_flagstat" executable: "target/executable/samtools/samtools_flagstat/samtools_flagstat" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/samtools/samtools_flagstat/samtools_flagstat b/target/executable/samtools/samtools_flagstat/samtools_flagstat index c715ce64..01e9c6c6 100755 --- a/target/executable/samtools/samtools_flagstat/samtools_flagstat +++ b/target/executable/samtools/samtools_flagstat/samtools_flagstat @@ -2,7 +2,7 @@ # samtools_flagstat main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -57,24 +57,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -213,9 +213,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -267,9 +267,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -289,7 +289,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -313,9 +314,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -473,9 +474,9 @@ sed 's#Using ##;s# \([0-9\.]*\)$#: \1#' > /var/software_versions.txt LABEL org.opencontainers.image.authors="Emma Rousseau" LABEL org.opencontainers.image.description="Companion container for running component samtools samtools_flagstat" -LABEL org.opencontainers.image.created="2024-08-21T11:35:53Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:41Z" LABEL org.opencontainers.image.source="https://github.com/samtools/samtools" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -502,6 +503,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -532,14 +536,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -547,7 +554,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -555,7 +562,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -563,17 +572,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -657,6 +671,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -718,6 +744,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/samtools/samtools_idxstats/.config.vsh.yaml b/target/executable/samtools/samtools_idxstats/.config.vsh.yaml index 9e86142c..58885680 100644 --- a/target/executable/samtools/samtools_idxstats/.config.vsh.yaml +++ b/target/executable/samtools/samtools_idxstats/.config.vsh.yaml @@ -182,15 +182,15 @@ build_info: engine: "docker|native" output: "target/executable/samtools/samtools_idxstats" executable: "target/executable/samtools/samtools_idxstats/samtools_idxstats" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/samtools/samtools_idxstats/samtools_idxstats b/target/executable/samtools/samtools_idxstats/samtools_idxstats index 683a6eec..3b1b2d90 100755 --- a/target/executable/samtools/samtools_idxstats/samtools_idxstats +++ b/target/executable/samtools/samtools_idxstats/samtools_idxstats @@ -2,7 +2,7 @@ # samtools_idxstats main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -57,24 +57,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -217,9 +217,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -271,9 +271,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -293,7 +293,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -317,9 +318,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -477,9 +478,9 @@ sed 's#Using ##;s# \([0-9\.]*\)$#: \1#' > /var/software_versions.txt LABEL org.opencontainers.image.authors="Emma Rousseau" LABEL org.opencontainers.image.description="Companion container for running component samtools samtools_idxstats" -LABEL org.opencontainers.image.created="2024-08-21T11:35:52Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:41Z" LABEL org.opencontainers.image.source="https://github.com/samtools/samtools" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -506,6 +507,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -536,14 +540,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -551,7 +558,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -559,7 +566,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -567,17 +576,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -672,6 +686,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -733,6 +759,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/samtools/samtools_index/.config.vsh.yaml b/target/executable/samtools/samtools_index/.config.vsh.yaml index 4ba2537d..5e534797 100644 --- a/target/executable/samtools/samtools_index/.config.vsh.yaml +++ b/target/executable/samtools/samtools_index/.config.vsh.yaml @@ -188,15 +188,15 @@ build_info: engine: "docker|native" output: "target/executable/samtools/samtools_index" executable: "target/executable/samtools/samtools_index/samtools_index" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/samtools/samtools_index/samtools_index b/target/executable/samtools/samtools_index/samtools_index index 83e9bf11..1db07a04 100755 --- a/target/executable/samtools/samtools_index/samtools_index +++ b/target/executable/samtools/samtools_index/samtools_index @@ -2,7 +2,7 @@ # samtools_index main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -57,24 +57,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -224,9 +224,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -278,9 +278,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -300,7 +300,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -324,9 +325,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -484,9 +485,9 @@ sed 's#Using ##;s# \([0-9\.]*\)$#: \1#' > /var/software_versions.txt LABEL org.opencontainers.image.authors="Emma Rousseau" LABEL org.opencontainers.image.description="Companion container for running component samtools samtools_index" -LABEL org.opencontainers.image.created="2024-08-21T11:35:52Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:40Z" LABEL org.opencontainers.image.source="https://github.com/samtools/samtools" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -513,6 +514,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -543,14 +547,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -558,7 +565,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -566,7 +573,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -574,17 +583,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -700,6 +714,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -761,6 +787,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/samtools/samtools_sort/.config.vsh.yaml b/target/executable/samtools/samtools_sort/.config.vsh.yaml index af1e162c..2a9359ce 100644 --- a/target/executable/samtools/samtools_sort/.config.vsh.yaml +++ b/target/executable/samtools/samtools_sort/.config.vsh.yaml @@ -331,15 +331,15 @@ build_info: engine: "docker|native" output: "target/executable/samtools/samtools_sort" executable: "target/executable/samtools/samtools_sort/samtools_sort" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/samtools/samtools_sort/samtools_sort b/target/executable/samtools/samtools_sort/samtools_sort index ea44e2cb..c499205e 100755 --- a/target/executable/samtools/samtools_sort/samtools_sort +++ b/target/executable/samtools/samtools_sort/samtools_sort @@ -2,7 +2,7 @@ # samtools_sort main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -57,24 +57,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -295,9 +295,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -349,9 +349,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -371,7 +371,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -395,9 +396,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -555,9 +556,9 @@ sed 's#Using ##;s# \([0-9\.]*\)$#: \1#' > /var/software_versions.txt LABEL org.opencontainers.image.authors="Emma Rousseau" LABEL org.opencontainers.image.description="Companion container for running component samtools samtools_sort" -LABEL org.opencontainers.image.created="2024-08-21T11:35:55Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:43Z" LABEL org.opencontainers.image.source="https://github.com/samtools/samtools" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -584,6 +585,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -614,14 +618,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -629,7 +636,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -637,7 +644,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -645,17 +654,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -955,6 +969,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -1016,6 +1042,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/samtools/samtools_stats/.config.vsh.yaml b/target/executable/samtools/samtools_stats/.config.vsh.yaml index 441c07f1..4f413103 100644 --- a/target/executable/samtools/samtools_stats/.config.vsh.yaml +++ b/target/executable/samtools/samtools_stats/.config.vsh.yaml @@ -400,15 +400,15 @@ build_info: engine: "docker|native" output: "target/executable/samtools/samtools_stats" executable: "target/executable/samtools/samtools_stats/samtools_stats" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/samtools/samtools_stats/samtools_stats b/target/executable/samtools/samtools_stats/samtools_stats index a637c4e4..1a8dfd01 100755 --- a/target/executable/samtools/samtools_stats/samtools_stats +++ b/target/executable/samtools/samtools_stats/samtools_stats @@ -2,7 +2,7 @@ # samtools_stats main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -57,24 +57,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -314,9 +314,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -368,9 +368,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -390,7 +390,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -414,9 +415,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -574,9 +575,9 @@ sed 's#Using ##;s# \([0-9\.]*\)$#: \1#' > /var/software_versions.txt LABEL org.opencontainers.image.authors="Emma Rousseau" LABEL org.opencontainers.image.description="Companion container for running component samtools samtools_stats" -LABEL org.opencontainers.image.created="2024-08-21T11:35:55Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:43Z" LABEL org.opencontainers.image.source="https://github.com/samtools/samtools" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -603,6 +604,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -633,14 +637,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -648,7 +655,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -656,7 +663,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -664,17 +673,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -1078,6 +1092,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -1139,6 +1165,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/samtools/samtools_view/.config.vsh.yaml b/target/executable/samtools/samtools_view/.config.vsh.yaml index 66e3768e..bc1f5e2c 100644 --- a/target/executable/samtools/samtools_view/.config.vsh.yaml +++ b/target/executable/samtools/samtools_view/.config.vsh.yaml @@ -664,15 +664,15 @@ build_info: engine: "docker|native" output: "target/executable/samtools/samtools_view" executable: "target/executable/samtools/samtools_view/samtools_view" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/samtools/samtools_view/samtools_view b/target/executable/samtools/samtools_view/samtools_view index 9def3cc3..d70dcfef 100755 --- a/target/executable/samtools/samtools_view/samtools_view +++ b/target/executable/samtools/samtools_view/samtools_view @@ -2,7 +2,7 @@ # samtools_view main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -57,24 +57,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -564,9 +564,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -618,9 +618,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -640,7 +640,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -664,9 +665,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -824,9 +825,9 @@ sed 's#Using ##;s# \([0-9\.]*\)$#: \1#' > /var/software_versions.txt LABEL org.opencontainers.image.authors="Emma Rousseau" LABEL org.opencontainers.image.description="Companion container for running component samtools samtools_view" -LABEL org.opencontainers.image.created="2024-08-21T11:35:53Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:41Z" LABEL org.opencontainers.image.source="https://github.com/samtools/samtools" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -853,6 +854,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -883,14 +887,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -898,7 +905,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -906,7 +913,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -914,17 +923,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -1556,6 +1570,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -1617,6 +1643,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/seqtk/seqtk_sample/.config.vsh.yaml b/target/executable/seqtk/seqtk_sample/.config.vsh.yaml index f96f6d3b..a2efe08f 100644 --- a/target/executable/seqtk/seqtk_sample/.config.vsh.yaml +++ b/target/executable/seqtk/seqtk_sample/.config.vsh.yaml @@ -172,15 +172,15 @@ build_info: engine: "docker|native" output: "target/executable/seqtk/seqtk_sample" executable: "target/executable/seqtk/seqtk_sample/seqtk_sample" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/seqtk/seqtk_sample/seqtk_sample b/target/executable/seqtk/seqtk_sample/seqtk_sample index fbed128a..71995762 100755 --- a/target/executable/seqtk/seqtk_sample/seqtk_sample +++ b/target/executable/seqtk/seqtk_sample/seqtk_sample @@ -2,7 +2,7 @@ # seqtk_sample main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -57,24 +57,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -223,9 +223,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -277,9 +277,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -299,7 +299,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -323,9 +324,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -480,9 +481,9 @@ FROM quay.io/biocontainers/seqtk:1.4--he4a0461_2 ENTRYPOINT [] LABEL org.opencontainers.image.authors="Jakub Majercik" LABEL org.opencontainers.image.description="Companion container for running component seqtk seqtk_sample" -LABEL org.opencontainers.image.created="2024-08-21T11:36:05Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:54Z" LABEL org.opencontainers.image.source="https://github.com/lh3/seqtk/tree/v1.4" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -509,6 +510,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -539,14 +543,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -554,7 +561,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -562,7 +569,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -570,17 +579,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -680,6 +694,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -741,6 +767,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/seqtk/seqtk_subseq/.config.vsh.yaml b/target/executable/seqtk/seqtk_subseq/.config.vsh.yaml index 14f425b6..41fdc969 100644 --- a/target/executable/seqtk/seqtk_subseq/.config.vsh.yaml +++ b/target/executable/seqtk/seqtk_subseq/.config.vsh.yaml @@ -195,15 +195,15 @@ build_info: engine: "docker|native" output: "target/executable/seqtk/seqtk_subseq" executable: "target/executable/seqtk/seqtk_subseq/seqtk_subseq" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/seqtk/seqtk_subseq/seqtk_subseq b/target/executable/seqtk/seqtk_subseq/seqtk_subseq index 70ad29aa..e905ada9 100755 --- a/target/executable/seqtk/seqtk_subseq/seqtk_subseq +++ b/target/executable/seqtk/seqtk_subseq/seqtk_subseq @@ -2,7 +2,7 @@ # seqtk_subseq main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -57,24 +57,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -231,9 +231,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -285,9 +285,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -307,7 +307,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -331,9 +332,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -490,9 +491,9 @@ RUN echo $(echo $(seqtk 2>&1) | sed -n 's/.*\(Version: [^ ]*\).*/\1/p') > /var/s LABEL org.opencontainers.image.authors="Theodoro Gasperin Terra Camargo" LABEL org.opencontainers.image.description="Companion container for running component seqtk seqtk_subseq" -LABEL org.opencontainers.image.created="2024-08-21T11:36:05Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:54Z" LABEL org.opencontainers.image.source="https://github.com/lh3/seqtk/tree/v1.4" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -519,6 +520,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -549,14 +553,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -564,7 +571,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -572,7 +579,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -580,17 +589,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -717,6 +731,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -778,6 +804,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/star/star_align_reads/.config.vsh.yaml b/target/executable/star/star_align_reads/.config.vsh.yaml index bd6707d4..9d92f349 100644 --- a/target/executable/star/star_align_reads/.config.vsh.yaml +++ b/target/executable/star/star_align_reads/.config.vsh.yaml @@ -2662,15 +2662,15 @@ build_info: engine: "docker|native" output: "target/executable/star/star_align_reads" executable: "target/executable/star/star_align_reads/star_align_reads" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/star/star_align_reads/star_align_reads b/target/executable/star/star_align_reads/star_align_reads index 391fdccb..36a893c3 100755 --- a/target/executable/star/star_align_reads/star_align_reads +++ b/target/executable/star/star_align_reads/star_align_reads @@ -2,7 +2,7 @@ # star_align_reads main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -58,24 +58,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -1638,9 +1638,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -1692,9 +1692,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -1714,7 +1714,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -1738,9 +1739,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -1919,9 +1920,9 @@ RUN STAR --version | sed 's#\(.*\)#star: "\1"#' > /var/software_versions.txt LABEL org.opencontainers.image.authors="Angela Oliveira Pisco, Robrecht Cannoodt" LABEL org.opencontainers.image.description="Companion container for running component star star_align_reads" -LABEL org.opencontainers.image.created="2024-08-21T11:35:58Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:46Z" LABEL org.opencontainers.image.source="https://github.com/alexdobin/STAR" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -1948,6 +1949,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -1978,14 +1982,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -1993,7 +2000,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -2001,7 +2008,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -2009,17 +2018,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -4462,6 +4476,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -4523,6 +4549,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/star/star_genome_generate/.config.vsh.yaml b/target/executable/star/star_genome_generate/.config.vsh.yaml index 2b739f42..e75e9a95 100644 --- a/target/executable/star/star_genome_generate/.config.vsh.yaml +++ b/target/executable/star/star_genome_generate/.config.vsh.yaml @@ -332,15 +332,15 @@ build_info: engine: "docker|native" output: "target/executable/star/star_genome_generate" executable: "target/executable/star/star_genome_generate/star_genome_generate" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/star/star_genome_generate/star_genome_generate b/target/executable/star/star_genome_generate/star_genome_generate index 3ebc41be..62df966b 100755 --- a/target/executable/star/star_genome_generate/star_genome_generate +++ b/target/executable/star/star_genome_generate/star_genome_generate @@ -2,7 +2,7 @@ # star_genome_generate main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -57,24 +57,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -302,9 +302,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -356,9 +356,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -378,7 +378,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -402,9 +403,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -576,9 +577,9 @@ RUN STAR --version | sed 's#\(.*\)#star: "\1"#' > /var/software_versions.txt LABEL org.opencontainers.image.authors="Sai Nirmayi Yasa" LABEL org.opencontainers.image.description="Companion container for running component star star_genome_generate" -LABEL org.opencontainers.image.created="2024-08-21T11:35:58Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:46Z" LABEL org.opencontainers.image.source="https://github.com/alexdobin/STAR" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -605,6 +606,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -635,14 +639,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -650,7 +657,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -658,7 +665,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -666,17 +675,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -932,6 +946,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -993,6 +1019,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/umi_tools/umi_tools_dedup/.config.vsh.yaml b/target/executable/umi_tools/umi_tools_dedup/.config.vsh.yaml index afd0164e..aedd11ec 100644 --- a/target/executable/umi_tools/umi_tools_dedup/.config.vsh.yaml +++ b/target/executable/umi_tools/umi_tools_dedup/.config.vsh.yaml @@ -610,15 +610,15 @@ build_info: engine: "docker|native" output: "target/executable/umi_tools/umi_tools_dedup" executable: "target/executable/umi_tools/umi_tools_dedup/umi_tools_dedup" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/umi_tools/umi_tools_dedup/umi_tools_dedup b/target/executable/umi_tools/umi_tools_dedup/umi_tools_dedup index b99ca5b3..2d38ffb5 100755 --- a/target/executable/umi_tools/umi_tools_dedup/umi_tools_dedup +++ b/target/executable/umi_tools/umi_tools_dedup/umi_tools_dedup @@ -2,7 +2,7 @@ # umi_tools_dedup main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -57,24 +57,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -510,9 +510,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -564,9 +564,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -586,7 +586,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -610,9 +611,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -769,9 +770,9 @@ RUN umi_tools -v | sed 's/ version//g' > /var/software_versions.txt LABEL org.opencontainers.image.authors="Emma Rousseau" LABEL org.opencontainers.image.description="Companion container for running component umi_tools umi_tools_dedup" -LABEL org.opencontainers.image.created="2024-08-21T11:35:56Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:44Z" LABEL org.opencontainers.image.source="https://github.com/CGATOxford/UMI-tools" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -798,6 +799,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -828,14 +832,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -843,7 +850,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -851,7 +858,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -859,17 +868,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -1395,6 +1409,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -1456,6 +1482,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/executable/umi_tools/umi_tools_extract/.config.vsh.yaml b/target/executable/umi_tools/umi_tools_extract/.config.vsh.yaml index 19fc3e13..6e483307 100644 --- a/target/executable/umi_tools/umi_tools_extract/.config.vsh.yaml +++ b/target/executable/umi_tools/umi_tools_extract/.config.vsh.yaml @@ -448,15 +448,15 @@ build_info: engine: "docker|native" output: "target/executable/umi_tools/umi_tools_extract" executable: "target/executable/umi_tools/umi_tools_extract/umi_tools_extract" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/executable/umi_tools/umi_tools_extract/umi_tools_extract b/target/executable/umi_tools/umi_tools_extract/umi_tools_extract index 61a66eba..e0398cb3 100755 --- a/target/executable/umi_tools/umi_tools_extract/umi_tools_extract +++ b/target/executable/umi_tools/umi_tools_extract/umi_tools_extract @@ -2,7 +2,7 @@ # umi_tools_extract main # -# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a # derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from # Data Intuitive. # @@ -54,24 +54,24 @@ function ViashRemoveFlags { # $1 : Should always be set to ${BASH_SOURCE[0]} # returns : The absolute path of the bash file function ViashSourceDir { - SOURCE="$1" - while [ -h "$SOURCE" ]; do - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" done - cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd } # ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks # usage : ViashFindTargetDir 'ScriptPath' # $1 : The location from where to start the upward search # returns : The absolute path of the '.build.yaml' file function ViashFindTargetDir { - SOURCE="$1" - while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do - SOURCE=${SOURCE%/*} + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} done - echo $SOURCE + echo $source } # see https://en.wikipedia.org/wiki/Syslog#Severity_level VIASH_LOGCODE_EMERGENCY=0 @@ -378,9 +378,9 @@ function ViashDockerInstallationCheck { fi ViashDebug "Checking whether the Docker daemon is running" - save=$-; set +e - docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) - out=$? + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashCritical "Docker daemon does not seem to be running. Try one of the following:" @@ -432,9 +432,9 @@ function ViashDockerPull { if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker pull $1 && return 0 || return 1 else - save=$-; set +e + local save=$-; set +e docker pull $1 2> /dev/null > /dev/null - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." @@ -454,7 +454,8 @@ function ViashDockerPull { # echo $? # returns '1' function ViashDockerPush { ViashNotice "Pushing image to '$1'" - save=$-; set +e + local save=$-; set +e + local out if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then docker push $1 out=$? @@ -478,9 +479,9 @@ function ViashDockerPush { # examples: # ViashDockerPullElseBuild mynewcomponent function ViashDockerPullElseBuild { - save=$-; set +e + local save=$-; set +e ViashDockerPull $1 - out=$? + local out=$? [[ $save =~ e ]] && set -e if [ $out -ne 0 ]; then ViashDockerBuild $@ @@ -636,9 +637,9 @@ ENTRYPOINT [] RUN umi_tools -v | sed 's/ version//g' > /var/software_versions.txt LABEL org.opencontainers.image.description="Companion container for running component umi_tools umi_tools_extract" -LABEL org.opencontainers.image.created="2024-08-21T11:35:56Z" +LABEL org.opencontainers.image.created="2024-09-02T13:02:44Z" LABEL org.opencontainers.image.source="https://github.com/CGATOxford/UMI-tools" -LABEL org.opencontainers.image.revision="766ab6c9c3059004c7c3f205621909b2d8b0b26d" +LABEL org.opencontainers.image.revision="2b29a47575db9dbdff8448b287925c25d9a8b01d" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -665,6 +666,9 @@ function ViashDockerBuildArgs { # ViashAbsolutePath /foo/bar/.. # returns /foo function ViashAbsolutePath { local thePath + local parr + local outp + local len if [[ ! "$1" =~ ^/ ]]; then thePath="$PWD/$1" else @@ -695,14 +699,17 @@ function ViashAbsolutePath { ) } # ViashDockerAutodetectMount: auto configuring docker mounts from parameters -# $1 : The parameter value -# returns : New parameter -# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts # examples: # ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' # ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' function ViashDockerAutodetectMount { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -710,7 +717,7 @@ function ViashDockerAutodetectMount { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" if [ -z "$base_name" ]; then echo "$mount_target" else @@ -718,7 +725,9 @@ function ViashDockerAutodetectMount { fi } function ViashDockerAutodetectMountArg { - abs_path=$(ViashAbsolutePath "$1") + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name if [ -d "$abs_path" ]; then mount_source="$abs_path" base_name="" @@ -726,17 +735,22 @@ function ViashDockerAutodetectMountArg { mount_source=`dirname "$abs_path"` base_name=`basename "$abs_path"` fi - mount_target="/viash_automount$mount_source" + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashDockerStripAutomount { - abs_path=$(ViashAbsolutePath "$1") - echo "${abs_path#/viash_automount}" + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" } # initialise variables VIASH_DIRECTORY_MOUNTS=() +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) @@ -1109,6 +1123,18 @@ while [[ $# -gt 0 ]]; do VIASH_MODE='dockerfile' shift 1 ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; ---debug) VIASH_MODE='debug' shift 1 @@ -1170,6 +1196,10 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ "$VIASH_MODE" == "dockerfile" ]; then ViashDockerfile "$VIASH_ENGINE_ID" exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 # enter docker container elif [[ "$VIASH_MODE" == "debug" ]]; then diff --git a/target/nextflow/agat/agat_convert_bed2gff/.config.vsh.yaml b/target/nextflow/agat/agat_convert_bed2gff/.config.vsh.yaml index ba20050a..0a97c243 100644 --- a/target/nextflow/agat/agat_convert_bed2gff/.config.vsh.yaml +++ b/target/nextflow/agat/agat_convert_bed2gff/.config.vsh.yaml @@ -234,15 +234,15 @@ build_info: engine: "docker|native" output: "target/nextflow/agat/agat_convert_bed2gff" executable: "target/nextflow/agat/agat_convert_bed2gff/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/agat/agat_convert_bed2gff/main.nf b/target/nextflow/agat/agat_convert_bed2gff/main.nf index 1a9d0c14..b76e8688 100644 --- a/target/nextflow/agat/agat_convert_bed2gff/main.nf +++ b/target/nextflow/agat/agat_convert_bed2gff/main.nf @@ -1,6 +1,6 @@ // agat_convert_bed2gff main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -763,8 +763,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -786,7 +789,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -824,8 +840,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1601,8 +1620,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2629,30 +2648,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2723,7 +2743,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -3065,15 +3085,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/agat/agat_convert_bed2gff", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/agat/agat_convert_embl2gff/.config.vsh.yaml b/target/nextflow/agat/agat_convert_embl2gff/.config.vsh.yaml index b4be3e72..5793a549 100644 --- a/target/nextflow/agat/agat_convert_embl2gff/.config.vsh.yaml +++ b/target/nextflow/agat/agat_convert_embl2gff/.config.vsh.yaml @@ -224,15 +224,15 @@ build_info: engine: "docker|native" output: "target/nextflow/agat/agat_convert_embl2gff" executable: "target/nextflow/agat/agat_convert_embl2gff/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/agat/agat_convert_embl2gff/main.nf b/target/nextflow/agat/agat_convert_embl2gff/main.nf index 8e73dc48..d74d8074 100644 --- a/target/nextflow/agat/agat_convert_embl2gff/main.nf +++ b/target/nextflow/agat/agat_convert_embl2gff/main.nf @@ -1,6 +1,6 @@ // agat_convert_embl2gff main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -763,8 +763,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -786,7 +789,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -824,8 +840,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1601,8 +1620,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2629,30 +2648,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2723,7 +2743,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -3058,15 +3078,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/agat/agat_convert_embl2gff", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/agat/agat_convert_embl2gff/nextflow_schema.json b/target/nextflow/agat/agat_convert_embl2gff/nextflow_schema.json index c3c2b98f..b959bda2 100644 --- a/target/nextflow/agat/agat_convert_embl2gff/nextflow_schema.json +++ b/target/nextflow/agat/agat_convert_embl2gff/nextflow_schema.json @@ -69,8 +69,8 @@ "primary_tag": { "type": "string", - "description": "Type: List of `string`, example: `tag1:tag2`, multiple_sep: `\":\"`. List of \"primary tag\"", - "help_text": "Type: List of `string`, example: `tag1:tag2`, multiple_sep: `\":\"`. List of \"primary tag\". Useful to discard or keep specific features. Multiple tags must be comma-separated.\n" + "description": "Type: List of `string`, example: `tag1;tag2`, multiple_sep: `\";\"`. List of \"primary tag\"", + "help_text": "Type: List of `string`, example: `tag1;tag2`, multiple_sep: `\";\"`. List of \"primary tag\". Useful to discard or keep specific features. Multiple tags must be comma-separated.\n" } diff --git a/target/nextflow/agat/agat_convert_sp_gff2gtf/.config.vsh.yaml b/target/nextflow/agat/agat_convert_sp_gff2gtf/.config.vsh.yaml index 50247564..c344ee97 100644 --- a/target/nextflow/agat/agat_convert_sp_gff2gtf/.config.vsh.yaml +++ b/target/nextflow/agat/agat_convert_sp_gff2gtf/.config.vsh.yaml @@ -227,15 +227,15 @@ build_info: engine: "docker|native" output: "target/nextflow/agat/agat_convert_sp_gff2gtf" executable: "target/nextflow/agat/agat_convert_sp_gff2gtf/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/agat/agat_convert_sp_gff2gtf/main.nf b/target/nextflow/agat/agat_convert_sp_gff2gtf/main.nf index 09ec6503..df675a96 100644 --- a/target/nextflow/agat/agat_convert_sp_gff2gtf/main.nf +++ b/target/nextflow/agat/agat_convert_sp_gff2gtf/main.nf @@ -1,6 +1,6 @@ // agat_convert_sp_gff2gtf main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -763,8 +763,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -786,7 +789,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -824,8 +840,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1601,8 +1620,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2629,30 +2648,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2723,7 +2743,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -3041,15 +3061,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/agat/agat_convert_sp_gff2gtf", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/agat/agat_convert_sp_gff2tsv/.config.vsh.yaml b/target/nextflow/agat/agat_convert_sp_gff2tsv/.config.vsh.yaml index 00d06376..59d2028a 100644 --- a/target/nextflow/agat/agat_convert_sp_gff2tsv/.config.vsh.yaml +++ b/target/nextflow/agat/agat_convert_sp_gff2tsv/.config.vsh.yaml @@ -187,15 +187,15 @@ build_info: engine: "docker|native" output: "target/nextflow/agat/agat_convert_sp_gff2tsv" executable: "target/nextflow/agat/agat_convert_sp_gff2tsv/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/agat/agat_convert_sp_gff2tsv/main.nf b/target/nextflow/agat/agat_convert_sp_gff2tsv/main.nf index f9bcbbd5..9d785588 100644 --- a/target/nextflow/agat/agat_convert_sp_gff2tsv/main.nf +++ b/target/nextflow/agat/agat_convert_sp_gff2tsv/main.nf @@ -1,6 +1,6 @@ // agat_convert_sp_gff2tsv main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -763,8 +763,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -786,7 +789,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -824,8 +840,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1601,8 +1620,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2629,30 +2648,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2723,7 +2743,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -3019,15 +3039,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/agat/agat_convert_sp_gff2tsv", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/agat/agat_convert_sp_gxf2gxf/.config.vsh.yaml b/target/nextflow/agat/agat_convert_sp_gxf2gxf/.config.vsh.yaml index 9101e247..9c4ccfbf 100644 --- a/target/nextflow/agat/agat_convert_sp_gxf2gxf/.config.vsh.yaml +++ b/target/nextflow/agat/agat_convert_sp_gxf2gxf/.config.vsh.yaml @@ -194,15 +194,15 @@ build_info: engine: "docker|native" output: "target/nextflow/agat/agat_convert_sp_gxf2gxf" executable: "target/nextflow/agat/agat_convert_sp_gxf2gxf/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/agat/agat_convert_sp_gxf2gxf/main.nf b/target/nextflow/agat/agat_convert_sp_gxf2gxf/main.nf index f0a12b36..9970339d 100644 --- a/target/nextflow/agat/agat_convert_sp_gxf2gxf/main.nf +++ b/target/nextflow/agat/agat_convert_sp_gxf2gxf/main.nf @@ -1,6 +1,6 @@ // agat_convert_sp_gxf2gxf main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -763,8 +763,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -786,7 +789,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -824,8 +840,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1601,8 +1620,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2629,30 +2648,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2723,7 +2743,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -3019,15 +3039,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/agat/agat_convert_sp_gxf2gxf", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/arriba/.config.vsh.yaml b/target/nextflow/arriba/.config.vsh.yaml index ea0bdd83..76ab3915 100644 --- a/target/nextflow/arriba/.config.vsh.yaml +++ b/target/nextflow/arriba/.config.vsh.yaml @@ -705,15 +705,15 @@ build_info: engine: "docker|native" output: "target/nextflow/arriba" executable: "target/nextflow/arriba/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/arriba/main.nf b/target/nextflow/arriba/main.nf index a9c482b3..89fb803b 100644 --- a/target/nextflow/arriba/main.nf +++ b/target/nextflow/arriba/main.nf @@ -1,6 +1,6 @@ // arriba main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -763,8 +763,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -786,7 +789,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -824,8 +840,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1601,8 +1620,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2629,30 +2648,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2723,7 +2743,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -3565,15 +3585,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/arriba", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/arriba/nextflow_schema.json b/target/nextflow/arriba/nextflow_schema.json index d9b5968e..bc31a94b 100644 --- a/target/nextflow/arriba/nextflow_schema.json +++ b/target/nextflow/arriba/nextflow_schema.json @@ -161,8 +161,8 @@ "interesting_contigs": { "type": "string", - "description": "Type: List of `string`, example: `1:2:AC_*:NC_*`, multiple_sep: `\":\"`. List of interesting contigs", - "help_text": "Type: List of `string`, example: `1:2:AC_*:NC_*`, multiple_sep: `\":\"`. List of interesting contigs. Fusions between genes \non other contigs are ignored. Contigs can be specified with or without the \nprefix \"chr\". Asterisks (*) are treated as wild-cards. \nDefault: 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 X Y AC_* NC_*\n" + "description": "Type: List of `string`, example: `1;2;AC_*;NC_*`, multiple_sep: `\";\"`. List of interesting contigs", + "help_text": "Type: List of `string`, example: `1;2;AC_*;NC_*`, multiple_sep: `\";\"`. List of interesting contigs. Fusions between genes \non other contigs are ignored. Contigs can be specified with or without the \nprefix \"chr\". Asterisks (*) are treated as wild-cards. \nDefault: 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 X Y AC_* NC_*\n" } @@ -171,8 +171,8 @@ "viral_contigs": { "type": "string", - "description": "Type: List of `string`, example: `AC_*:NC_*`, multiple_sep: `\":\"`. List of viral contigs", - "help_text": "Type: List of `string`, example: `AC_*:NC_*`, multiple_sep: `\":\"`. List of viral contigs. Asterisks (*) are treated as \nwild-cards.\nDefault: AC_* NC_*\n" + "description": "Type: List of `string`, example: `AC_*;NC_*`, multiple_sep: `\";\"`. List of viral contigs", + "help_text": "Type: List of `string`, example: `AC_*;NC_*`, multiple_sep: `\";\"`. List of viral contigs. Asterisks (*) are treated as \nwild-cards.\nDefault: AC_* NC_*\n" } @@ -181,8 +181,8 @@ "disable_filters": { "type": "string", - "description": "Type: List of `string`, multiple_sep: `\":\"`, choices: ``homologs`, `low_entropy`, `isoforms`, `top_expressed_viral_contigs`, `viral_contigs`, `uninteresting_contigs`, `non_coding_neighbors`, `mismatches`, `duplicates`, `no_genomic_support`, `genomic_support`, `intronic`, `end_to_end`, `relative_support`, `low_coverage_viral_contigs`, `merge_adjacent`, `mismappers`, `multimappers`, `same_gene`, `long_gap`, `internal_tandem_duplication`, `small_insert_size`, `read_through`, `inconsistently_clipped`, `intragenic_exonic`, `marginal_read_through`, `spliced`, `hairpin`, `blacklist`, `min_support`, `select_best`, `in_vitro`, `short_anchor`, `known_fusions`, `no_coverage`, `homopolymer`, `many_spliced``. List of filters to disable", - "help_text": "Type: List of `string`, multiple_sep: `\":\"`, choices: ``homologs`, `low_entropy`, `isoforms`, `top_expressed_viral_contigs`, `viral_contigs`, `uninteresting_contigs`, `non_coding_neighbors`, `mismatches`, `duplicates`, `no_genomic_support`, `genomic_support`, `intronic`, `end_to_end`, `relative_support`, `low_coverage_viral_contigs`, `merge_adjacent`, `mismappers`, `multimappers`, `same_gene`, `long_gap`, `internal_tandem_duplication`, `small_insert_size`, `read_through`, `inconsistently_clipped`, `intragenic_exonic`, `marginal_read_through`, `spliced`, `hairpin`, `blacklist`, `min_support`, `select_best`, `in_vitro`, `short_anchor`, `known_fusions`, `no_coverage`, `homopolymer`, `many_spliced``. List of filters to disable. By default all filters are \nenabled. \n", + "description": "Type: List of `string`, multiple_sep: `\";\"`, choices: ``homologs`, `low_entropy`, `isoforms`, `top_expressed_viral_contigs`, `viral_contigs`, `uninteresting_contigs`, `non_coding_neighbors`, `mismatches`, `duplicates`, `no_genomic_support`, `genomic_support`, `intronic`, `end_to_end`, `relative_support`, `low_coverage_viral_contigs`, `merge_adjacent`, `mismappers`, `multimappers`, `same_gene`, `long_gap`, `internal_tandem_duplication`, `small_insert_size`, `read_through`, `inconsistently_clipped`, `intragenic_exonic`, `marginal_read_through`, `spliced`, `hairpin`, `blacklist`, `min_support`, `select_best`, `in_vitro`, `short_anchor`, `known_fusions`, `no_coverage`, `homopolymer`, `many_spliced``. List of filters to disable", + "help_text": "Type: List of `string`, multiple_sep: `\";\"`, choices: ``homologs`, `low_entropy`, `isoforms`, `top_expressed_viral_contigs`, `viral_contigs`, `uninteresting_contigs`, `non_coding_neighbors`, `mismatches`, `duplicates`, `no_genomic_support`, `genomic_support`, `intronic`, `end_to_end`, `relative_support`, `low_coverage_viral_contigs`, `merge_adjacent`, `mismappers`, `multimappers`, `same_gene`, `long_gap`, `internal_tandem_duplication`, `small_insert_size`, `read_through`, `inconsistently_clipped`, `intragenic_exonic`, `marginal_read_through`, `spliced`, `hairpin`, `blacklist`, `min_support`, `select_best`, `in_vitro`, `short_anchor`, `known_fusions`, `no_coverage`, `homopolymer`, `many_spliced``. List of filters to disable. By default all filters are \nenabled. \n", "enum": ["homologs", "low_entropy", "isoforms", "top_expressed_viral_contigs", "viral_contigs", "uninteresting_contigs", "non_coding_neighbors", "mismatches", "duplicates", "no_genomic_support", "genomic_support", "intronic", "end_to_end", "relative_support", "low_coverage_viral_contigs", "merge_adjacent", "mismappers", "multimappers", "same_gene", "long_gap", "internal_tandem_duplication", "small_insert_size", "read_through", "inconsistently_clipped", "intragenic_exonic", "marginal_read_through", "spliced", "hairpin", "blacklist", "min_support", "select_best", "in_vitro", "short_anchor", "known_fusions", "no_coverage", "homopolymer", "many_spliced"] diff --git a/target/nextflow/bcl_convert/.config.vsh.yaml b/target/nextflow/bcl_convert/.config.vsh.yaml index bbed5a1e..c42b8645 100644 --- a/target/nextflow/bcl_convert/.config.vsh.yaml +++ b/target/nextflow/bcl_convert/.config.vsh.yaml @@ -417,15 +417,15 @@ build_info: engine: "docker|native" output: "target/nextflow/bcl_convert" executable: "target/nextflow/bcl_convert/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/bcl_convert/main.nf b/target/nextflow/bcl_convert/main.nf index 32f94280..eb277261 100644 --- a/target/nextflow/bcl_convert/main.nf +++ b/target/nextflow/bcl_convert/main.nf @@ -1,6 +1,6 @@ // bcl_convert main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -764,8 +764,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -787,7 +790,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -825,8 +841,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1602,8 +1621,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2630,30 +2649,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2724,7 +2744,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -3302,15 +3322,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/bcl_convert", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/bd_rhapsody/bd_rhapsody_make_reference/.config.vsh.yaml b/target/nextflow/bd_rhapsody/bd_rhapsody_make_reference/.config.vsh.yaml index 96f7a8de..b6848744 100644 --- a/target/nextflow/bd_rhapsody/bd_rhapsody_make_reference/.config.vsh.yaml +++ b/target/nextflow/bd_rhapsody/bd_rhapsody_make_reference/.config.vsh.yaml @@ -269,15 +269,15 @@ build_info: engine: "docker|native" output: "target/nextflow/bd_rhapsody/bd_rhapsody_make_reference" executable: "target/nextflow/bd_rhapsody/bd_rhapsody_make_reference/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/bd_rhapsody/bd_rhapsody_make_reference/main.nf b/target/nextflow/bd_rhapsody/bd_rhapsody_make_reference/main.nf index 9cf164be..2e7dd448 100644 --- a/target/nextflow/bd_rhapsody/bd_rhapsody_make_reference/main.nf +++ b/target/nextflow/bd_rhapsody/bd_rhapsody_make_reference/main.nf @@ -1,6 +1,6 @@ // bd_rhapsody_make_reference main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -764,8 +764,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -787,7 +790,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -825,8 +841,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1602,8 +1621,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2630,30 +2649,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2724,7 +2744,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -3115,15 +3135,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/bd_rhapsody/bd_rhapsody_make_reference", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/bd_rhapsody/bd_rhapsody_make_reference/nextflow_schema.json b/target/nextflow/bd_rhapsody/bd_rhapsody_make_reference/nextflow_schema.json index 1cf13642..10a2e002 100644 --- a/target/nextflow/bd_rhapsody/bd_rhapsody_make_reference/nextflow_schema.json +++ b/target/nextflow/bd_rhapsody/bd_rhapsody_make_reference/nextflow_schema.json @@ -17,8 +17,8 @@ "genome_fasta": { "type": "string", - "description": "Type: List of `file`, required, example: `genome_sequence.fa.gz`, multiple_sep: `\":\"`. Reference genome file in FASTA or FASTA", - "help_text": "Type: List of `file`, required, example: `genome_sequence.fa.gz`, multiple_sep: `\":\"`. Reference genome file in FASTA or FASTA.GZ format. The BD Rhapsody Sequencing Analysis Pipeline uses GRCh38 for Human and GRCm39 for Mouse." + "description": "Type: List of `file`, required, example: `genome_sequence.fa.gz`, multiple_sep: `\";\"`. Reference genome file in FASTA or FASTA", + "help_text": "Type: List of `file`, required, example: `genome_sequence.fa.gz`, multiple_sep: `\";\"`. Reference genome file in FASTA or FASTA.GZ format. The BD Rhapsody Sequencing Analysis Pipeline uses GRCh38 for Human and GRCm39 for Mouse." } @@ -27,8 +27,8 @@ "gtf": { "type": "string", - "description": "Type: List of `file`, required, example: `transcriptome_annotation.gtf.gz`, multiple_sep: `\":\"`. File path to the transcript annotation files in GTF or GTF", - "help_text": "Type: List of `file`, required, example: `transcriptome_annotation.gtf.gz`, multiple_sep: `\":\"`. File path to the transcript annotation files in GTF or GTF.GZ format. The Sequence Analysis Pipeline requires the \u0027gene_name\u0027 or \n\u0027gene_id\u0027 attribute to be set on each gene and exon feature. Gene and exon feature lines must have the same attribute, and exons\nmust have a corresponding gene with the same value. For TCR/BCR assays, the TCR or BCR gene segments must have the \u0027gene_type\u0027 or\n\u0027gene_biotype\u0027 attribute set, and the value should begin with \u0027TR\u0027 or \u0027IG\u0027, respectively.\n" + "description": "Type: List of `file`, required, example: `transcriptome_annotation.gtf.gz`, multiple_sep: `\";\"`. File path to the transcript annotation files in GTF or GTF", + "help_text": "Type: List of `file`, required, example: `transcriptome_annotation.gtf.gz`, multiple_sep: `\";\"`. File path to the transcript annotation files in GTF or GTF.GZ format. The Sequence Analysis Pipeline requires the \u0027gene_name\u0027 or \n\u0027gene_id\u0027 attribute to be set on each gene and exon feature. Gene and exon feature lines must have the same attribute, and exons\nmust have a corresponding gene with the same value. For TCR/BCR assays, the TCR or BCR gene segments must have the \u0027gene_type\u0027 or\n\u0027gene_biotype\u0027 attribute set, and the value should begin with \u0027TR\u0027 or \u0027IG\u0027, respectively.\n" } @@ -37,8 +37,8 @@ "extra_sequences": { "type": "string", - "description": "Type: List of `file`, multiple_sep: `\":\"`. File path to additional sequences in FASTA format to use when building the STAR index", - "help_text": "Type: List of `file`, multiple_sep: `\":\"`. File path to additional sequences in FASTA format to use when building the STAR index. (e.g. transgenes or CRISPR guide barcodes).\nGTF lines for these sequences will be automatically generated and combined with the main GTF.\n" + "description": "Type: List of `file`, multiple_sep: `\";\"`. File path to additional sequences in FASTA format to use when building the STAR index", + "help_text": "Type: List of `file`, multiple_sep: `\";\"`. File path to additional sequences in FASTA format to use when building the STAR index. (e.g. transgenes or CRISPR guide barcodes).\nGTF lines for these sequences will be automatically generated and combined with the main GTF.\n" } @@ -78,10 +78,10 @@ "mitochondrial_contigs": { "type": "string", - "description": "Type: List of `string`, default: `chrM:chrMT:M:MT`, multiple_sep: `\":\"`. Names of the Mitochondrial contigs in the provided Reference Genome", - "help_text": "Type: List of `string`, default: `chrM:chrMT:M:MT`, multiple_sep: `\":\"`. Names of the Mitochondrial contigs in the provided Reference Genome. Fragments originating from contigs other than these are\nidentified as \u0027nuclear fragments\u0027 in the ATACseq analysis pipeline.\n" + "description": "Type: List of `string`, default: `chrM;chrMT;M;MT`, multiple_sep: `\";\"`. Names of the Mitochondrial contigs in the provided Reference Genome", + "help_text": "Type: List of `string`, default: `chrM;chrMT;M;MT`, multiple_sep: `\";\"`. Names of the Mitochondrial contigs in the provided Reference Genome. Fragments originating from contigs other than these are\nidentified as \u0027nuclear fragments\u0027 in the ATACseq analysis pipeline.\n" , - "default": "chrM:chrMT:M:MT" + "default": "chrM;chrMT;M;MT" } diff --git a/target/nextflow/bedtools/bedtools_bamtofastq/.config.vsh.yaml b/target/nextflow/bedtools/bedtools_bamtofastq/.config.vsh.yaml index c830f6c5..f60e552b 100644 --- a/target/nextflow/bedtools/bedtools_bamtofastq/.config.vsh.yaml +++ b/target/nextflow/bedtools/bedtools_bamtofastq/.config.vsh.yaml @@ -186,15 +186,15 @@ build_info: engine: "docker|native" output: "target/nextflow/bedtools/bedtools_bamtofastq" executable: "target/nextflow/bedtools/bedtools_bamtofastq/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/bedtools/bedtools_bamtofastq/main.nf b/target/nextflow/bedtools/bedtools_bamtofastq/main.nf index a839ee9b..024d72f6 100644 --- a/target/nextflow/bedtools/bedtools_bamtofastq/main.nf +++ b/target/nextflow/bedtools/bedtools_bamtofastq/main.nf @@ -1,6 +1,6 @@ // bedtools_bamtofastq main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -763,8 +763,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -786,7 +789,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -824,8 +840,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1601,8 +1620,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2629,30 +2648,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2723,7 +2743,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -3023,15 +3043,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/bedtools/bedtools_bamtofastq", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/bedtools/bedtools_bedtobam/.config.vsh.yaml b/target/nextflow/bedtools/bedtools_bedtobam/.config.vsh.yaml index 6430b5a8..4c5a4ba7 100644 --- a/target/nextflow/bedtools/bedtools_bedtobam/.config.vsh.yaml +++ b/target/nextflow/bedtools/bedtools_bedtobam/.config.vsh.yaml @@ -213,15 +213,15 @@ build_info: engine: "docker|native" output: "target/nextflow/bedtools/bedtools_bedtobam" executable: "target/nextflow/bedtools/bedtools_bedtobam/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/bedtools/bedtools_bedtobam/main.nf b/target/nextflow/bedtools/bedtools_bedtobam/main.nf index e1522b33..264fbaaf 100644 --- a/target/nextflow/bedtools/bedtools_bedtobam/main.nf +++ b/target/nextflow/bedtools/bedtools_bedtobam/main.nf @@ -1,6 +1,6 @@ // bedtools_bedtobam main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -763,8 +763,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -786,7 +789,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -824,8 +840,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1601,8 +1620,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2629,30 +2648,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2723,7 +2743,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -3056,15 +3076,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/bedtools/bedtools_bedtobam", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/bedtools/bedtools_getfasta/.config.vsh.yaml b/target/nextflow/bedtools/bedtools_getfasta/.config.vsh.yaml index 6b575393..ec6aeb43 100644 --- a/target/nextflow/bedtools/bedtools_getfasta/.config.vsh.yaml +++ b/target/nextflow/bedtools/bedtools_getfasta/.config.vsh.yaml @@ -231,15 +231,15 @@ build_info: engine: "docker|native" output: "target/nextflow/bedtools/bedtools_getfasta" executable: "target/nextflow/bedtools/bedtools_getfasta/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/bedtools/bedtools_getfasta/main.nf b/target/nextflow/bedtools/bedtools_getfasta/main.nf index 99019b44..c32cfcbe 100644 --- a/target/nextflow/bedtools/bedtools_getfasta/main.nf +++ b/target/nextflow/bedtools/bedtools_getfasta/main.nf @@ -1,6 +1,6 @@ // bedtools_getfasta main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -763,8 +763,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -786,7 +789,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -824,8 +840,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1601,8 +1620,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2629,30 +2648,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2723,7 +2743,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -3059,15 +3079,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/bedtools/bedtools_getfasta", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/bedtools/bedtools_groupby/.config.vsh.yaml b/target/nextflow/bedtools/bedtools_groupby/.config.vsh.yaml new file mode 100644 index 00000000..60a93533 --- /dev/null +++ b/target/nextflow/bedtools/bedtools_groupby/.config.vsh.yaml @@ -0,0 +1,299 @@ +name: "bedtools_groupby" +namespace: "bedtools" +version: "main" +authors: +- name: "Theodoro Gasperin Terra Camargo" + roles: + - "author" + - "maintainer" + info: + links: + email: "theodorogtc@gmail.com" + github: "tgaspe" + linkedin: "theodoro-gasperin-terra-camargo" + organizations: + - name: "Data Intuitive" + href: "https://www.data-intuitive.com" + role: "Bioinformatician" +argument_groups: +- name: "Inputs" + arguments: + - type: "file" + name: "--input" + alternatives: + - "-i" + description: "The input BED file to be used.\n" + info: null + example: + - "input_a.bed" + must_exist: true + create_parent: true + required: true + direction: "input" + multiple: false + multiple_sep: ";" +- name: "Outputs" + arguments: + - type: "file" + name: "--output" + description: "The output groupby BED file. \n" + info: null + example: + - "output.bed" + must_exist: true + create_parent: true + required: true + direction: "output" + multiple: false + multiple_sep: ";" +- name: "Options" + arguments: + - type: "string" + name: "--groupby" + alternatives: + - "-g" + - "-grp" + description: "Specify the columns (1-based) for the grouping.\nThe columns must\ + \ be comma separated.\n- Default: 1,2,3 \n" + info: null + required: true + direction: "input" + multiple: false + multiple_sep: ";" + - type: "integer" + name: "--column" + alternatives: + - "-c" + - "-opCols" + description: "Specify the column (1-based) that should be summarized.\n" + info: null + required: true + direction: "input" + multiple: false + multiple_sep: ";" + - type: "string" + name: "--operation" + alternatives: + - "-o" + - "-ops" + description: "Specify the operation that should be applied to opCol.\nValid operations:\n\ + \ sum, count, count_distinct, min, max,\n mean, median, mode, antimode,\n\ + \ stdev, sstdev (sample standard dev.),\n collapse (i.e., print a comma\ + \ separated list (duplicates allowed)), \n distinct (i.e., print a comma\ + \ separated list (NO duplicates allowed)), \n distinct_sort_num (as distinct,\ + \ but sorted numerically, ascending), \n distinct_sort_num_desc (as distinct,\ + \ but sorted numerically, descending), \n concat (i.e., merge values into\ + \ a single, non-delimited string), \n freqdesc (i.e., print desc. list of\ + \ values:freq)\n freqasc (i.e., print asc. list of values:freq)\n first\ + \ (i.e., print first value)\n last (i.e., print last value)\n\nDefault value:\ + \ sum \n\nIf there is only column, but multiple operations, all operations\ + \ will be\napplied on that column. Likewise, if there is only one operation,\ + \ but\nmultiple columns, that operation will be applied to all columns.\nOtherwise,\ + \ the number of columns must match the the number of operations,\nand will be\ + \ applied in respective order.\nE.g., \"-c 5,4,6 -o sum,mean,count\" will give\ + \ the sum of column 5,\nthe mean of column 4, and the count of column 6.\nThe\ + \ order of output columns will match the ordering given in the command.\n" + info: null + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "boolean_true" + name: "--full" + description: "Print all columns from input file. The first line in the group is\ + \ used.\nDefault: print only grouped columns.\n" + info: null + direction: "input" + - type: "boolean_true" + name: "--inheader" + description: "Input file has a header line - the first line will be ignored.\n" + info: null + direction: "input" + - type: "boolean_true" + name: "--outheader" + description: "Print header line in the output, detailing the column names. \n\ + If the input file has headers (-inheader), the output file\nwill use the input's\ + \ column names.\nIf the input file has no headers, the output file\nwill use\ + \ \"col_1\", \"col_2\", etc. as the column names.\n" + info: null + direction: "input" + - type: "boolean_true" + name: "--header" + description: "same as '-inheader -outheader'." + info: null + direction: "input" + - type: "boolean_true" + name: "--ignorecase" + description: "Group values regardless of upper/lower case.\n" + info: null + direction: "input" + - type: "integer" + name: "--precision" + alternatives: + - "-prec" + description: "Sets the decimal precision for output. \n" + info: null + default: + - 5 + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "string" + name: "--delimiter" + alternatives: + - "-delim" + description: "Specify a custom delimiter for the collapse operations.\n" + info: null + example: + - "|" + default: + - "," + required: false + direction: "input" + multiple: false + multiple_sep: ";" +resources: +- type: "bash_script" + path: "script.sh" + is_executable: true +description: "Summarizes a dataset column based upon common column groupings. \nAkin\ + \ to the SQL \"group by\" command.\n" +test_resources: +- type: "bash_script" + path: "test.sh" + is_executable: true +info: null +status: "enabled" +requirements: + commands: + - "ps" +keywords: +- "groupby" +- "BED" +license: "MIT" +references: + doi: + - "10.1093/bioinformatics/btq033" +links: + repository: "https://github.com/arq5x/bedtools2" + homepage: "https://bedtools.readthedocs.io/en/latest/#" + documentation: "https://bedtools.readthedocs.io/en/latest/content/tools/groupby.html" + issue_tracker: "https://github.com/arq5x/bedtools2/issues" +runners: +- type: "executable" + id: "executable" + docker_setup_strategy: "ifneedbepullelsecachedbuild" +- type: "nextflow" + id: "nextflow" + directives: + tag: "$id" + auto: + simplifyInput: true + simplifyOutput: false + transcript: false + publish: false + config: + labels: + mem1gb: "memory = 1000000000.B" + mem2gb: "memory = 2000000000.B" + mem5gb: "memory = 5000000000.B" + mem10gb: "memory = 10000000000.B" + mem20gb: "memory = 20000000000.B" + mem50gb: "memory = 50000000000.B" + mem100gb: "memory = 100000000000.B" + mem200gb: "memory = 200000000000.B" + mem500gb: "memory = 500000000000.B" + mem1tb: "memory = 1000000000000.B" + mem2tb: "memory = 2000000000000.B" + mem5tb: "memory = 5000000000000.B" + mem10tb: "memory = 10000000000000.B" + mem20tb: "memory = 20000000000000.B" + mem50tb: "memory = 50000000000000.B" + mem100tb: "memory = 100000000000000.B" + mem200tb: "memory = 200000000000000.B" + mem500tb: "memory = 500000000000000.B" + mem1gib: "memory = 1073741824.B" + mem2gib: "memory = 2147483648.B" + mem4gib: "memory = 4294967296.B" + mem8gib: "memory = 8589934592.B" + mem16gib: "memory = 17179869184.B" + mem32gib: "memory = 34359738368.B" + mem64gib: "memory = 68719476736.B" + mem128gib: "memory = 137438953472.B" + mem256gib: "memory = 274877906944.B" + mem512gib: "memory = 549755813888.B" + mem1tib: "memory = 1099511627776.B" + mem2tib: "memory = 2199023255552.B" + mem4tib: "memory = 4398046511104.B" + mem8tib: "memory = 8796093022208.B" + mem16tib: "memory = 17592186044416.B" + mem32tib: "memory = 35184372088832.B" + mem64tib: "memory = 70368744177664.B" + mem128tib: "memory = 140737488355328.B" + mem256tib: "memory = 281474976710656.B" + mem512tib: "memory = 562949953421312.B" + cpu1: "cpus = 1" + cpu2: "cpus = 2" + cpu5: "cpus = 5" + cpu10: "cpus = 10" + cpu20: "cpus = 20" + cpu50: "cpus = 50" + cpu100: "cpus = 100" + cpu200: "cpus = 200" + cpu500: "cpus = 500" + cpu1000: "cpus = 1000" + debug: false + container: "docker" +engines: +- type: "docker" + id: "docker" + image: "debian:stable-slim" + target_registry: "images.viash-hub.com" + target_tag: "main" + namespace_separator: "/" + setup: + - type: "apt" + packages: + - "bedtools" + - "procps" + interactive: false + - type: "docker" + run: + - "echo \"bedtools: \\\"$(bedtools --version | sed -n 's/^bedtools //p')\\\"\"\ + \ > /var/software_versions.txt\n" + entrypoint: [] + cmd: null +- type: "native" + id: "native" +build_info: + config: "src/bedtools/bedtools_groupby/config.vsh.yaml" + runner: "nextflow" + engine: "docker|native" + output: "target/nextflow/bedtools/bedtools_groupby" + executable: "target/nextflow/bedtools/bedtools_groupby/main.nf" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" + git_remote: "https://github.com/viash-hub/biobox" +package_config: + name: "biobox" + version: "main" + description: "A collection of bioinformatics tools for working with sequence data.\n" + info: null + viash_version: "0.9.0-RC7" + source: "src" + target: "target" + config_mods: + - ".requirements.commands := ['ps']\n" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'main'" + keywords: + - "bioinformatics" + - "modules" + - "sequencing" + license: "MIT" + organization: "vsh" + links: + repository: "https://github.com/viash-hub/biobox" + issue_tracker: "https://github.com/viash-hub/biobox/issues" diff --git a/target/nextflow/bedtools/bedtools_groupby/main.nf b/target/nextflow/bedtools/bedtools_groupby/main.nf new file mode 100644 index 00000000..565ee15a --- /dev/null +++ b/target/nextflow/bedtools/bedtools_groupby/main.nf @@ -0,0 +1,3710 @@ +// bedtools_groupby main +// +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a +// derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from +// Data Intuitive. +// +// The component may contain files which fall under a different license. The +// authors of this component should specify the license in the header of such +// files, or include a separate license file detailing the licenses of all included +// files. +// +// Component authors: +// * Theodoro Gasperin Terra Camargo (author, maintainer) + +//////////////////////////// +// VDSL3 helper functions // +//////////////////////////// + +// helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_checkArgumentType.nf' +class UnexpectedArgumentTypeException extends Exception { + String errorIdentifier + String stage + String plainName + String expectedClass + String foundClass + + // ${key ? " in module '$key'" : ""}${id ? " id '$id'" : ""} + UnexpectedArgumentTypeException(String errorIdentifier, String stage, String plainName, String expectedClass, String foundClass) { + super("Error${errorIdentifier ? " $errorIdentifier" : ""}:${stage ? " $stage" : "" } argument '${plainName}' has the wrong type. " + + "Expected type: ${expectedClass}. Found type: ${foundClass}") + this.errorIdentifier = errorIdentifier + this.stage = stage + this.plainName = plainName + this.expectedClass = expectedClass + this.foundClass = foundClass + } +} + +/** + * Checks if the given value is of the expected type. If not, an exception is thrown. + * + * @param stage The stage of the argument (input or output) + * @param par The parameter definition + * @param value The value to check + * @param errorIdentifier The identifier to use in the error message + * @return The value, if it is of the expected type + * @throws UnexpectedArgumentTypeException If the value is not of the expected type +*/ +def _checkArgumentType(String stage, Map par, Object value, String errorIdentifier) { + // expectedClass will only be != null if value is not of the expected type + def expectedClass = null + def foundClass = null + + // todo: split if need be + + if (!par.required && value == null) { + expectedClass = null + } else if (par.multiple) { + if (value !instanceof Collection) { + value = [value] + } + + // split strings + value = value.collectMany{ val -> + if (val instanceof String) { + // collect() to ensure that the result is a List and not simply an array + val.split(par.multiple_sep).collect() + } else { + [val] + } + } + + // process globs + if (par.type == "file" && par.direction == "input") { + value = value.collect{ it instanceof String ? file(it, hidden: true) : it }.flatten() + } + + // check types of elements in list + try { + value = value.collect { listVal -> + _checkArgumentType(stage, par + [multiple: false], listVal, errorIdentifier) + } + } catch (UnexpectedArgumentTypeException e) { + expectedClass = "List[${e.expectedClass}]" + foundClass = "List[${e.foundClass}]" + } + } else if (par.type == "string") { + // cast to string if need be + if (value instanceof GString) { + value = value.toString() + } + expectedClass = value instanceof String ? null : "String" + } else if (par.type == "integer") { + // cast to integer if need be + if (value instanceof String) { + try { + value = value.toInteger() + } catch (NumberFormatException e) { + // do nothing + } + } + if (value instanceof java.math.BigInteger) { + value = value.intValue() + } + expectedClass = value instanceof Integer ? null : "Integer" + } else if (par.type == "long") { + // cast to long if need be + if (value instanceof String) { + try { + value = value.toLong() + } catch (NumberFormatException e) { + // do nothing + } + } + if (value instanceof Integer) { + value = value.toLong() + } + expectedClass = value instanceof Long ? null : "Long" + } else if (par.type == "double") { + // cast to double if need be + if (value instanceof String) { + try { + value = value.toDouble() + } catch (NumberFormatException e) { + // do nothing + } + } + if (value instanceof java.math.BigDecimal) { + value = value.doubleValue() + } + if (value instanceof Float) { + value = value.toDouble() + } + expectedClass = value instanceof Double ? null : "Double" + } else if (par.type == "boolean" | par.type == "boolean_true" | par.type == "boolean_false") { + // cast to boolean if need be + if (value instanceof String) { + def valueLower = value.toLowerCase() + if (valueLower == "true") { + value = true + } else if (valueLower == "false") { + value = false + } + } + expectedClass = value instanceof Boolean ? null : "Boolean" + } else if (par.type == "file" && (par.direction == "input" || stage == "output")) { + // cast to path if need be + if (value instanceof String) { + value = file(value, hidden: true) + } + if (value instanceof File) { + value = value.toPath() + } + expectedClass = value instanceof Path ? null : "Path" + } else if (par.type == "file" && stage == "input" && par.direction == "output") { + // cast to string if need be + if (value instanceof GString) { + value = value.toString() + } + expectedClass = value instanceof String ? null : "String" + } else { + // didn't find a match for par.type + expectedClass = par.type + } + + if (expectedClass != null) { + if (foundClass == null) { + foundClass = value.getClass().getName() + } + throw new UnexpectedArgumentTypeException(errorIdentifier, stage, par.plainName, expectedClass, foundClass) + } + + return value +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_processInputValues.nf' +Map _processInputValues(Map inputs, Map config, String id, String key) { + if (!workflow.stubRun) { + config.allArguments.each { arg -> + if (arg.required) { + assert inputs.containsKey(arg.plainName) && inputs.get(arg.plainName) != null : + "Error in module '${key}' id '${id}': required input argument '${arg.plainName}' is missing" + } + } + + inputs = inputs.collectEntries { name, value -> + def par = config.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") } + assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid input argument" + + value = _checkArgumentType("input", par, value, "in module '$key' id '$id'") + + [ name, value ] + } + } + return inputs +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_processOutputValues.nf' +Map _processOutputValues(Map outputs, Map config, String id, String key) { + if (!workflow.stubRun) { + config.allArguments.each { arg -> + if (arg.direction == "output" && arg.required) { + assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : + "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing" + } + } + + outputs = outputs.collectEntries { name, value -> + def par = config.allArguments.find { it.plainName == name && it.direction == "output" } + assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid output argument" + + value = _checkArgumentType("output", par, value, "in module '$key' id '$id'") + + [ name, value ] + } + } + return outputs +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/IDChecker.nf' +class IDChecker { + final def items = [] as Set + + @groovy.transform.WithWriteLock + boolean observe(String item) { + if (items.contains(item)) { + return false + } else { + items << item + return true + } + } + + @groovy.transform.WithReadLock + boolean contains(String item) { + return items.contains(item) + } + + @groovy.transform.WithReadLock + Set getItems() { + return items.clone() + } +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_checkUniqueIds.nf' + +/** + * Check if the ids are unique across parameter sets + * + * @param parameterSets a list of parameter sets. + */ +private void _checkUniqueIds(List>> parameterSets) { + def ppIds = parameterSets.collect{it[0]} + assert ppIds.size() == ppIds.unique().size() : "All argument sets should have unique ids. Detected ids: $ppIds" +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_getChild.nf' + +// helper functions for reading params from file // +def _getChild(parent, child) { + if (child.contains("://") || java.nio.file.Paths.get(child).isAbsolute()) { + child + } else { + def parentAbsolute = java.nio.file.Paths.get(parent).toAbsolutePath().toString() + parentAbsolute.replaceAll('/[^/]*$', "/") + child + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_parseParamList.nf' +/** + * Figure out the param list format based on the file extension + * + * @param param_list A String containing the path to the parameter list file. + * + * @return A String containing the format of the parameter list file. + */ +def _paramListGuessFormat(param_list) { + if (param_list !instanceof String) { + "asis" + } else if (param_list.endsWith(".csv")) { + "csv" + } else if (param_list.endsWith(".json") || param_list.endsWith(".jsn")) { + "json" + } else if (param_list.endsWith(".yaml") || param_list.endsWith(".yml")) { + "yaml" + } else { + "yaml_blob" + } +} + + +/** + * Read the param list + * + * @param param_list One of the following: + * - A String containing the path to the parameter list file (csv, json or yaml), + * - A yaml blob of a list of maps (yaml_blob), + * - Or a groovy list of maps (asis). + * @param config A Map of the Viash configuration. + * + * @return A List of Maps containing the parameters. + */ +def _parseParamList(param_list, Map config) { + // first determine format by extension + def paramListFormat = _paramListGuessFormat(param_list) + + def paramListPath = (paramListFormat != "asis" && paramListFormat != "yaml_blob") ? + file(param_list, hidden: true) : + null + + // get the correct parser function for the detected params_list format + def paramSets = [] + if (paramListFormat == "asis") { + paramSets = param_list + } else if (paramListFormat == "yaml_blob") { + paramSets = readYamlBlob(param_list) + } else if (paramListFormat == "yaml") { + paramSets = readYaml(paramListPath) + } else if (paramListFormat == "json") { + paramSets = readJson(paramListPath) + } else if (paramListFormat == "csv") { + paramSets = readCsv(paramListPath) + } else { + error "Format of provided --param_list not recognised.\n" + + "Found: '$paramListFormat'.\n" + + "Expected: a csv file, a json file, a yaml file,\n" + + "a yaml blob or a groovy list of maps." + } + + // data checks + assert paramSets instanceof List: "--param_list should contain a list of maps" + for (value in paramSets) { + assert value instanceof Map: "--param_list should contain a list of maps" + } + + // id is argument + def idIsArgument = config.allArguments.any{it.plainName == "id"} + + // Reformat from List to List> by adding the ID as first element of a Tuple2 + paramSets = paramSets.collect({ data -> + def id = data.id + if (!idIsArgument) { + data = data.findAll{k, v -> k != "id"} + } + [id, data] + }) + + // Split parameters with 'multiple: true' + paramSets = paramSets.collect({ id, data -> + data = _splitParams(data, config) + [id, data] + }) + + // The paths of input files inside a param_list file may have been specified relatively to the + // location of the param_list file. These paths must be made absolute. + if (paramListPath) { + paramSets = paramSets.collect({ id, data -> + def new_data = data.collectEntries{ parName, parValue -> + def par = config.allArguments.find{it.plainName == parName} + if (par && par.type == "file" && par.direction == "input") { + if (parValue instanceof Collection) { + parValue = parValue.collectMany{path -> + def x = _resolveSiblingIfNotAbsolute(path, paramListPath) + x instanceof Collection ? x : [x] + } + } else { + parValue = _resolveSiblingIfNotAbsolute(parValue, paramListPath) + } + } + [parName, parValue] + } + [id, new_data] + }) + } + + return paramSets +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_splitParams.nf' +/** + * Split parameters for arguments that accept multiple values using their separator + * + * @param paramList A Map containing parameters to split. + * @param config A Map of the Viash configuration. This Map can be generated from the config file + * using the readConfig() function. + * + * @return A Map of parameters where the parameter values have been split into a list using + * their seperator. + */ +Map _splitParams(Map parValues, Map config){ + def parsedParamValues = parValues.collectEntries { parName, parValue -> + def parameterSettings = config.allArguments.find({it.plainName == parName}) + + if (!parameterSettings) { + // if argument is not found, do not alter + return [parName, parValue] + } + if (parameterSettings.multiple) { // Check if parameter can accept multiple values + if (parValue instanceof Collection) { + parValue = parValue.collect{it instanceof String ? it.split(parameterSettings.multiple_sep) : it } + } else if (parValue instanceof String) { + parValue = parValue.split(parameterSettings.multiple_sep) + } else if (parValue == null) { + parValue = [] + } else { + parValue = [ parValue ] + } + parValue = parValue.flatten() + } + // For all parameters check if multiple values are only passed for + // arguments that allow it. Quietly simplify lists of length 1. + if (!parameterSettings.multiple && parValue instanceof Collection) { + assert parValue.size() == 1 : + "Error: argument ${parName} has too many values.\n" + + " Expected amount: 1. Found: ${parValue.size()}" + parValue = parValue[0] + } + [parName, parValue] + } + return parsedParamValues +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/channelFromParams.nf' +/** + * Parse nextflow parameters based on settings defined in a viash config. + * Return a list of parameter sets, each parameter set corresponding to + * an event in a nextflow channel. The output from this function can be used + * with Channel.fromList to create a nextflow channel with Vdsl3 formatted + * events. + * + * This function performs: + * - A filtering of the params which can be found in the config file. + * - Process the params_list argument which allows a user to to initialise + * a Vsdl3 channel with multiple parameter sets. Possible formats are + * csv, json, yaml, or simply a yaml_blob. A csv should have column names + * which correspond to the different arguments of this pipeline. A json or a yaml + * file should be a list of maps, each of which has keys corresponding to the + * arguments of the pipeline. A yaml blob can also be passed directly as a parameter. + * When passing a csv, json or yaml, relative path names are relativized to the + * location of the parameter file. + * - Combine the parameter sets into a vdsl3 Channel. + * + * @param params Input parameters. Can optionaly contain a 'param_list' key that + * provides a list of arguments that can be split up into multiple events + * in the output channel possible formats of param_lists are: a csv file, + * json file, a yaml file or a yaml blob. Each parameters set (event) must + * have a unique ID. + * @param config A Map of the Viash configuration. This Map can be generated from the config file + * using the readConfig() function. + * + * @return A list of parameters with the first element of the event being + * the event ID and the second element containing a map of the parsed parameters. + */ + +private List>> _paramsToParamSets(Map params, Map config){ + // todo: fetch key from run args + def key_ = config.name + + /* parse regular parameters (not in param_list) */ + /*************************************************/ + def globalParams = config.allArguments + .findAll { params.containsKey(it.plainName) } + .collectEntries { [ it.plainName, params[it.plainName] ] } + def globalID = params.get("id", null) + + /* process params_list arguments */ + /*********************************/ + def paramList = params.containsKey("param_list") && params.param_list != null ? + params.param_list : [] + // if (paramList instanceof String) { + // paramList = [paramList] + // } + // def paramSets = paramList.collectMany{ _parseParamList(it, config) } + // TODO: be able to process param_list when it is a list of strings + def paramSets = _parseParamList(paramList, config) + if (paramSets.isEmpty()) { + paramSets = [[null, [:]]] + } + + /* combine arguments into channel */ + /**********************************/ + def processedParams = paramSets.indexed().collect{ index, tup -> + // Process ID + def id = tup[0] ?: globalID + + if (workflow.stubRun && !id) { + // if stub run, explicitly add an id if missing + id = "stub${index}" + } + assert id != null: "Each parameter set should have at least an 'id'" + + // Process params + def parValues = globalParams + tup[1] + // // Remove parameters which are null, if the default is also null + // parValues = parValues.collectEntries{paramName, paramValue -> + // parameterSettings = config.functionality.allArguments.find({it.plainName == paramName}) + // if ( paramValue != null || parameterSettings.get("default", null) != null ) { + // [paramName, paramValue] + // } + // } + parValues = parValues.collectEntries { name, value -> + def par = config.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") } + assert par != null : "Error in module '${key_}' id '${id}': '${name}' is not a valid input argument" + + if (par == null) { + return [:] + } + value = _checkArgumentType("input", par, value, "in module '$key_' id '$id'") + + [ name, value ] + } + + [id, parValues] + } + + // Check if ids (first element of each list) is unique + _checkUniqueIds(processedParams) + return processedParams +} + +/** + * Parse nextflow parameters based on settings defined in a viash config + * and return a nextflow channel. + * + * @param params Input parameters. Can optionaly contain a 'param_list' key that + * provides a list of arguments that can be split up into multiple events + * in the output channel possible formats of param_lists are: a csv file, + * json file, a yaml file or a yaml blob. Each parameters set (event) must + * have a unique ID. + * @param config A Map of the Viash configuration. This Map can be generated from the config file + * using the readConfig() function. + * + * @return A nextflow Channel with events. Events are formatted as a tuple that contains + * first contains the ID of the event and as second element holds a parameter map. + * + * + */ +def channelFromParams(Map params, Map config) { + def processedParams = _paramsToParamSets(params, config) + return Channel.fromList(processedParams) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/checkUniqueIds.nf' +def checkUniqueIds(Map args) { + def stopOnError = args.stopOnError == null ? args.stopOnError : true + + def idChecker = new IDChecker() + + return filter { tup -> + if (!idChecker.observe(tup[0])) { + if (stopOnError) { + error "Duplicate id: ${tup[0]}" + } else { + log.warn "Duplicate id: ${tup[0]}, removing duplicate entry" + return false + } + } + return true + } +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/preprocessInputs.nf' +// This helper file will be deprecated soon +preprocessInputsDeprecationWarningPrinted = false + +def preprocessInputsDeprecationWarning() { + if (!preprocessInputsDeprecationWarningPrinted) { + preprocessInputsDeprecationWarningPrinted = true + System.err.println("Warning: preprocessInputs() is deprecated and will be removed in Viash 0.9.0.") + } +} + +/** + * Generate a nextflow Workflow that allows processing a channel of + * Vdsl3 formatted events and apply a Viash config to them: + * - Gather default parameters from the Viash config and make + * sure that they are correctly formatted (see applyConfig method). + * - Format the input parameters (also using the applyConfig method). + * - Apply the default parameter to the input parameters. + * - Do some assertions: + * ~ Check if the event IDs in the channel are unique. + * + * The events in the channel are formatted as tuples, with the + * first element of the tuples being a unique id of the parameter set, + * and the second element containg the the parameters themselves. + * Optional extra elements of the tuples will be passed to the output as is. + * + * @param args A map that must contain a 'config' key that points + * to a parsed config (see readConfig()). Optionally, a + * 'key' key can be provided which can be used to create a unique + * name for the workflow process. + * + * @return A workflow that allows processing a channel of Vdsl3 formatted events + * and apply a Viash config to them. + */ +def preprocessInputs(Map args) { + preprocessInputsDeprecationWarning() + + def config = args.config + assert config instanceof Map : + "Error in preprocessInputs: config must be a map. " + + "Expected class: Map. Found: config.getClass() is ${config.getClass()}" + def key_ = args.key ?: config.name + + // Get different parameter types (used throughout this function) + def defaultArgs = config.allArguments + .findAll { it.containsKey("default") } + .collectEntries { [ it.plainName, it.default ] } + + map { tup -> + def id = tup[0] + def data = tup[1] + def passthrough = tup.drop(2) + + def new_data = (defaultArgs + data).collectEntries { name, value -> + def par = config.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") } + + if (par != null) { + value = _checkArgumentType("input", par, value, "in module '$key_' id '$id'") + } + + [ name, value ] + } + + [ id, new_data ] + passthrough + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/runComponents.nf' +/** + * Run a list of components on a stream of data. + * + * @param components: list of Viash VDSL3 modules to run + * @param fromState: a closure, a map or a list of keys to extract from the input data. + * If a closure, it will be called with the id, the data and the component config. + * @param toState: a closure, a map or a list of keys to extract from the output data + * If a closure, it will be called with the id, the output data, the old state and the component config. + * @param filter: filter function to apply to the input. + * It will be called with the id, the data and the component config. + * @param id: id to use for the output data + * If a closure, it will be called with the id, the data and the component config. + * @param auto: auto options to pass to the components + * + * @return: a workflow that runs the components + **/ +def runComponents(Map args) { + log.warn("runComponents is deprecated, use runEach instead") + assert args.components: "runComponents should be passed a list of components to run" + + def components_ = args.components + if (components_ !instanceof List) { + components_ = [ components_ ] + } + assert components_.size() > 0: "pass at least one component to runComponents" + + def fromState_ = args.fromState + def toState_ = args.toState + def filter_ = args.filter + def id_ = args.id + + workflow runComponentsWf { + take: input_ch + main: + + // generate one channel per method + out_chs = components_.collect{ comp_ -> + def comp_config = comp_.config + + def filter_ch = filter_ + ? input_ch | filter{tup -> + filter_(tup[0], tup[1], comp_config) + } + : input_ch + def id_ch = id_ + ? filter_ch | map{tup -> + // def new_id = id_(tup[0], tup[1], comp_config) + def new_id = tup[0] + if (id_ instanceof String) { + new_id = id_ + } else if (id_ instanceof Closure) { + new_id = id_(new_id, tup[1], comp_config) + } + [new_id] + tup.drop(1) + } + : filter_ch + def data_ch = id_ch | map{tup -> + def new_data = tup[1] + if (fromState_ instanceof Map) { + new_data = fromState_.collectEntries{ key0, key1 -> + [key0, new_data[key1]] + } + } else if (fromState_ instanceof List) { + new_data = fromState_.collectEntries{ key -> + [key, new_data[key]] + } + } else if (fromState_ instanceof Closure) { + new_data = fromState_(tup[0], new_data, comp_config) + } + tup.take(1) + [new_data] + tup.drop(1) + } + def out_ch = data_ch + | comp_.run( + auto: (args.auto ?: [:]) + [simplifyInput: false, simplifyOutput: false] + ) + def post_ch = toState_ + ? out_ch | map{tup -> + def output = tup[1] + def old_state = tup[2] + def new_state = null + if (toState_ instanceof Map) { + new_state = old_state + toState_.collectEntries{ key0, key1 -> + [key0, output[key1]] + } + } else if (toState_ instanceof List) { + new_state = old_state + toState_.collectEntries{ key -> + [key, output[key]] + } + } else if (toState_ instanceof Closure) { + new_state = toState_(tup[0], output, old_state, comp_config) + } + [tup[0], new_state] + tup.drop(3) + } + : out_ch + + post_ch + } + + // mix all results + output_ch = + (out_chs.size == 1) + ? out_chs[0] + : out_chs[0].mix(*out_chs.drop(1)) + + emit: output_ch + } + + return runComponentsWf +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/runEach.nf' +/** + * Run a list of components on a stream of data. + * + * @param components: list of Viash VDSL3 modules to run + * @param fromState: a closure, a map or a list of keys to extract from the input data. + * If a closure, it will be called with the id, the data and the component itself. + * @param toState: a closure, a map or a list of keys to extract from the output data + * If a closure, it will be called with the id, the output data, the old state and the component itself. + * @param filter: filter function to apply to the input. + * It will be called with the id, the data and the component itself. + * @param id: id to use for the output data + * If a closure, it will be called with the id, the data and the component itself. + * @param auto: auto options to pass to the components + * + * @return: a workflow that runs the components + **/ +def runEach(Map args) { + assert args.components: "runEach should be passed a list of components to run" + + def components_ = args.components + if (components_ !instanceof List) { + components_ = [ components_ ] + } + assert components_.size() > 0: "pass at least one component to runEach" + + def fromState_ = args.fromState + def toState_ = args.toState + def filter_ = args.filter + def runIf_ = args.runIf + def id_ = args.id + + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + + workflow runEachWf { + take: input_ch + main: + + // generate one channel per method + out_chs = components_.collect{ comp_ -> + def filter_ch = filter_ + ? input_ch | filter{tup -> + filter_(tup[0], tup[1], comp_) + } + : input_ch + def id_ch = id_ + ? filter_ch | map{tup -> + def new_id = id_ + if (new_id instanceof Closure) { + new_id = new_id(tup[0], tup[1], comp_) + } + assert new_id instanceof String : "Error in runEach: id should be a String or a Closure that returns a String. Expected: id instanceof String. Found: ${new_id.getClass()}" + [new_id] + tup.drop(1) + } + : filter_ch + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> + def new_data = tup[1] + if (fromState_ instanceof Map) { + new_data = fromState_.collectEntries{ key0, key1 -> + [key0, new_data[key1]] + } + } else if (fromState_ instanceof List) { + new_data = fromState_.collectEntries{ key -> + [key, new_data[key]] + } + } else if (fromState_ instanceof Closure) { + new_data = fromState_(tup[0], new_data, comp_) + } + tup.take(1) + [new_data] + tup.drop(1) + } + def out_ch = data_ch + | comp_.run( + auto: (args.auto ?: [:]) + [simplifyInput: false, simplifyOutput: false] + ) + def post_ch = toState_ + ? out_ch | map{tup -> + def output = tup[1] + def old_state = tup[2] + def new_state = null + if (toState_ instanceof Map) { + new_state = old_state + toState_.collectEntries{ key0, key1 -> + [key0, output[key1]] + } + } else if (toState_ instanceof List) { + new_state = old_state + toState_.collectEntries{ key -> + [key, output[key]] + } + } else if (toState_ instanceof Closure) { + new_state = toState_(tup[0], output, old_state, comp_) + } + [tup[0], new_state] + tup.drop(3) + } + : out_ch + + def return_ch = post_ch + | concat(chPassthrough) + + return_ch + } + + // mix all results + output_ch = + (out_chs.size == 1) + ? out_chs[0] + : out_chs[0].mix(*out_chs.drop(1)) + + emit: output_ch + } + + return runEachWf +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/safeJoin.nf' +/** + * Join sourceChannel to targetChannel + * + * This function joins the sourceChannel to the targetChannel. + * However, each id in the targetChannel must be present in the + * sourceChannel. If _meta.join_id exists in the targetChannel, that is + * used as an id instead. If the id doesn't match any id in the sourceChannel, + * an error is thrown. + */ + +def safeJoin(targetChannel, sourceChannel, key) { + def sourceIDs = new IDChecker() + + def sourceCheck = sourceChannel + | map { tup -> + sourceIDs.observe(tup[0]) + tup + } + def targetCheck = targetChannel + | map { tup -> + def id = tup[0] + + if (!sourceIDs.contains(id)) { + error ( + "Error in module '${key}' when merging output with original state.\n" + + " Reason: output with id '${id}' could not be joined with source channel.\n" + + " If the IDs in the output channel differ from the input channel,\n" + + " please set `tup[1]._meta.join_id to the original ID.\n" + + " Original IDs in input channel: ['${sourceIDs.getItems().join("', '")}'].\n" + + " Unexpected ID in the output channel: '${id}'.\n" + + " Example input event: [\"id\", [input: file(...)]],\n" + + " Example output event: [\"newid\", [output: file(...), _meta: [join_id: \"id\"]]]" + ) + } + // TODO: add link to our documentation on how to fix this + + tup + } + + sourceCheck.cross(targetChannel) + | map{ left, right -> + right + left.drop(1) + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/config/_processArgument.nf' +def _processArgument(arg) { + arg.multiple = arg.multiple != null ? arg.multiple : false + arg.required = arg.required != null ? arg.required : false + arg.direction = arg.direction != null ? arg.direction : "input" + arg.multiple_sep = arg.multiple_sep != null ? arg.multiple_sep : ";" + arg.plainName = arg.name.replaceAll("^-*", "") + + if (arg.type == "file") { + arg.must_exist = arg.must_exist != null ? arg.must_exist : true + arg.create_parent = arg.create_parent != null ? arg.create_parent : true + } + + // add default values to output files which haven't already got a default + if (arg.type == "file" && arg.direction == "output" && arg.default == null) { + def mult = arg.multiple ? "_*" : "" + def extSearch = "" + if (arg.default != null) { + extSearch = arg.default + } else if (arg.example != null) { + extSearch = arg.example + } + if (extSearch instanceof List) { + extSearch = extSearch[0] + } + def extSearchResult = extSearch.find("\\.[^\\.]+\$") + def ext = extSearchResult != null ? extSearchResult : "" + arg.default = "\$id.\$key.${arg.plainName}${mult}${ext}" + if (arg.multiple) { + arg.default = [arg.default] + } + } + + if (!arg.multiple) { + if (arg.default != null && arg.default instanceof List) { + arg.default = arg.default[0] + } + if (arg.example != null && arg.example instanceof List) { + arg.example = arg.example[0] + } + } + + if (arg.type == "boolean_true") { + arg.default = false + } + if (arg.type == "boolean_false") { + arg.default = true + } + + arg +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/config/addGlobalParams.nf' +def addGlobalArguments(config) { + def localConfig = [ + "argument_groups": [ + [ + "name": "Nextflow input-output arguments", + "description": "Input/output parameters for Nextflow itself. Please note that both publishDir and publish_dir are supported but at least one has to be configured.", + "arguments" : [ + [ + 'name': '--publish_dir', + 'required': true, + 'type': 'string', + 'description': 'Path to an output directory.', + 'example': 'output/', + 'multiple': false + ], + [ + 'name': '--param_list', + 'required': false, + 'type': 'string', + 'description': '''Allows inputting multiple parameter sets to initialise a Nextflow channel. A `param_list` can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. + | + |* A list of maps (as-is) where the keys of each map corresponds to the arguments of the pipeline. Example: in a `nextflow.config` file: `param_list: [ ['id': 'foo', 'input': 'foo.txt'], ['id': 'bar', 'input': 'bar.txt'] ]`. + |* A csv file should have column names which correspond to the different arguments of this pipeline. Example: `--param_list data.csv` with columns `id,input`. + |* A json or a yaml file should be a list of maps, each of which has keys corresponding to the arguments of the pipeline. Example: `--param_list data.json` with contents `[ {'id': 'foo', 'input': 'foo.txt'}, {'id': 'bar', 'input': 'bar.txt'} ]`. + |* A yaml blob can also be passed directly as a string. Example: `--param_list "[ {'id': 'foo', 'input': 'foo.txt'}, {'id': 'bar', 'input': 'bar.txt'} ]"`. + | + |When passing a csv, json or yaml file, relative path names are relativized to the location of the parameter file. No relativation is performed when `param_list` is a list of maps (as-is) or a yaml blob.'''.stripMargin(), + 'example': 'my_params.yaml', + 'multiple': false, + 'hidden': true + ] + // TODO: allow multiple: true in param_list? + // TODO: allow to specify a --param_list_regex to filter the param_list? + // TODO: allow to specify a --param_list_from_state to remap entries in the param_list? + ] + ] + ] + ] + + return processConfig(_mergeMap(config, localConfig)) +} + +def _mergeMap(Map lhs, Map rhs) { + return rhs.inject(lhs.clone()) { map, entry -> + if (map[entry.key] instanceof Map && entry.value instanceof Map) { + map[entry.key] = _mergeMap(map[entry.key], entry.value) + } else if (map[entry.key] instanceof Collection && entry.value instanceof Collection) { + map[entry.key] += entry.value + } else { + map[entry.key] = entry.value + } + return map + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/config/generateHelp.nf' +def _generateArgumentHelp(param) { + // alternatives are not supported + // def names = param.alternatives ::: List(param.name) + + def unnamedProps = [ + ["required parameter", param.required], + ["multiple values allowed", param.multiple], + ["output", param.direction.toLowerCase() == "output"], + ["file must exist", param.type == "file" && param.must_exist] + ].findAll{it[1]}.collect{it[0]} + + def dflt = null + if (param.default != null) { + if (param.default instanceof List) { + dflt = param.default.join(param.multiple_sep != null ? param.multiple_sep : ", ") + } else { + dflt = param.default.toString() + } + } + def example = null + if (param.example != null) { + if (param.example instanceof List) { + example = param.example.join(param.multiple_sep != null ? param.multiple_sep : ", ") + } else { + example = param.example.toString() + } + } + def min = param.min?.toString() + def max = param.max?.toString() + + def escapeChoice = { choice -> + def s1 = choice.replaceAll("\\n", "\\\\n") + def s2 = s1.replaceAll("\"", """\\\"""") + s2.contains(",") || s2 != choice ? "\"" + s2 + "\"" : s2 + } + def choices = param.choices == null ? + null : + "[ " + param.choices.collect{escapeChoice(it.toString())}.join(", ") + " ]" + + def namedPropsStr = [ + ["type", ([param.type] + unnamedProps).join(", ")], + ["default", dflt], + ["example", example], + ["choices", choices], + ["min", min], + ["max", max] + ] + .findAll{it[1]} + .collect{"\n " + it[0] + ": " + it[1].replaceAll("\n", "\\n")} + .join("") + + def descStr = param.description == null ? + "" : + _paragraphWrap("\n" + param.description.trim(), 80 - 8).join("\n ") + + "\n --" + param.plainName + + namedPropsStr + + descStr +} + +// Based on Helper.generateHelp() in Helper.scala +def _generateHelp(config) { + def fun = config + + // PART 1: NAME AND VERSION + def nameStr = fun.name + + (fun.version == null ? "" : " " + fun.version) + + // PART 2: DESCRIPTION + def descrStr = fun.description == null ? + "" : + "\n\n" + _paragraphWrap(fun.description.trim(), 80).join("\n") + + // PART 3: Usage + def usageStr = fun.usage == null ? + "" : + "\n\nUsage:\n" + fun.usage.trim() + + // PART 4: Options + def argGroupStrs = fun.allArgumentGroups.collect{argGroup -> + def name = argGroup.name + def descriptionStr = argGroup.description == null ? + "" : + "\n " + _paragraphWrap(argGroup.description.trim(), 80-4).join("\n ") + "\n" + def arguments = argGroup.arguments.collect{arg -> + arg instanceof String ? fun.allArguments.find{it.plainName == arg} : arg + }.findAll{it != null} + def argumentStrs = arguments.collect{param -> _generateArgumentHelp(param)} + + "\n\n$name:" + + descriptionStr + + argumentStrs.join("\n") + } + + // FINAL: combine + def out = nameStr + + descrStr + + usageStr + + argGroupStrs.join("") + + return out +} + +// based on Format._paragraphWrap +def _paragraphWrap(str, maxLength) { + def outLines = [] + str.split("\n").each{par -> + def words = par.split("\\s").toList() + + def word = null + def line = words.pop() + while(!words.isEmpty()) { + word = words.pop() + if (line.length() + word.length() + 1 <= maxLength) { + line = line + " " + word + } else { + outLines.add(line) + line = word + } + } + if (words.isEmpty()) { + outLines.add(line) + } + } + return outLines +} + +def helpMessage(config) { + if (params.containsKey("help") && params.help) { + def mergedConfig = addGlobalArguments(config) + def helpStr = _generateHelp(mergedConfig) + println(helpStr) + exit 0 + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/config/processConfig.nf' +def processConfig(config) { + // set defaults for arguments + config.arguments = + (config.arguments ?: []).collect{_processArgument(it)} + + // set defaults for argument_group arguments + config.argument_groups = + (config.argument_groups ?: []).collect{grp -> + grp.arguments = (grp.arguments ?: []).collect{_processArgument(it)} + grp + } + + // create combined arguments list + config.allArguments = + config.arguments + + config.argument_groups.collectMany{it.arguments} + + // add missing argument groups (based on Functionality::allArgumentGroups()) + def argGroups = config.argument_groups + if (argGroups.any{it.name.toLowerCase() == "arguments"}) { + argGroups = argGroups.collect{ grp -> + if (grp.name.toLowerCase() == "arguments") { + grp = grp + [ + arguments: grp.arguments + config.arguments + ] + } + grp + } + } else { + argGroups = argGroups + [ + name: "Arguments", + arguments: config.arguments + ] + } + config.allArgumentGroups = argGroups + + config +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/config/readConfig.nf' + +def readConfig(file) { + def config = readYaml(file ?: moduleDir.resolve("config.vsh.yaml")) + processConfig(config) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/_resolveSiblingIfNotAbsolute.nf' +/** + * Resolve a path relative to the current file. + * + * @param str The path to resolve, as a String. + * @param parentPath The path to resolve relative to, as a Path. + * + * @return The path that may have been resovled, as a Path. + */ +def _resolveSiblingIfNotAbsolute(str, parentPath) { + if (str !instanceof String) { + return str + } + if (!_stringIsAbsolutePath(str)) { + return parentPath.resolveSibling(str) + } else { + return file(str, hidden: true) + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/_stringIsAbsolutePath.nf' +/** + * Check whether a path as a string is absolute. + * + * In the past, we tried using `file(., relative: true).isAbsolute()`, + * but the 'relative' option was added in 22.10.0. + * + * @param path The path to check, as a String. + * + * @return Whether the path is absolute, as a boolean. + */ +def _stringIsAbsolutePath(path) { + def _resolve_URL_PROTOCOL = ~/^([a-zA-Z][a-zA-Z0-9]*:)?\\/.+/ + + assert path instanceof String + return _resolve_URL_PROTOCOL.matcher(path).matches() +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/collectTraces.nf' +class CustomTraceObserver implements nextflow.trace.TraceObserver { + List traces + + CustomTraceObserver(List traces) { + this.traces = traces + } + + @Override + void onProcessComplete(nextflow.processor.TaskHandler handler, nextflow.trace.TraceRecord trace) { + def trace2 = trace.store.clone() + trace2.script = null + traces.add(trace2) + } + + @Override + void onProcessCached(nextflow.processor.TaskHandler handler, nextflow.trace.TraceRecord trace) { + def trace2 = trace.store.clone() + trace2.script = null + traces.add(trace2) + } +} + +def collectTraces() { + def traces = Collections.synchronizedList([]) + + // add custom trace observer which stores traces in the traces object + session.observers.add(new CustomTraceObserver(traces)) + + traces +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/deepClone.nf' +/** + * Performs a deep clone of the given object. + * @param x an object + */ +def deepClone(x) { + iterateMap(x, {it instanceof Cloneable ? it.clone() : it}) +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/getPublishDir.nf' +def getPublishDir() { + return params.containsKey("publish_dir") ? params.publish_dir : + params.containsKey("publishDir") ? params.publishDir : + null +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/getRootDir.nf' + +// Recurse upwards until we find a '.build.yaml' file +def _findBuildYamlFile(pathPossiblySymlink) { + def path = pathPossiblySymlink.toRealPath() + def child = path.resolve(".build.yaml") + if (java.nio.file.Files.isDirectory(path) && java.nio.file.Files.exists(child)) { + return child + } else { + def parent = path.getParent() + if (parent == null) { + return null + } else { + return _findBuildYamlFile(parent) + } + } +} + +// get the root of the target folder +def getRootDir() { + def dir = _findBuildYamlFile(meta.resources_dir) + assert dir != null: "Could not find .build.yaml in the folder structure" + dir.getParent() +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/iterateMap.nf' +/** + * Recursively apply a function over the leaves of an object. + * @param obj The object to iterate over. + * @param fun The function to apply to each value. + * @return The object with the function applied to each value. + */ +def iterateMap(obj, fun) { + if (obj instanceof List && obj !instanceof String) { + return obj.collect{item -> + iterateMap(item, fun) + } + } else if (obj instanceof Map) { + return obj.collectEntries{key, item -> + [key.toString(), iterateMap(item, fun)] + } + } else { + return fun(obj) + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/niceView.nf' +/** + * A view for printing the event of each channel as a YAML blob. + * This is useful for debugging. + */ +def niceView() { + workflow niceViewWf { + take: input + main: + output = input + | view{toYamlBlob(it)} + emit: output + } + return niceViewWf +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readCsv.nf' + +def readCsv(file_path) { + def output = [] + def inputFile = file_path !instanceof Path ? file(file_path, hidden: true) : file_path + + // todo: allow escaped quotes in string + // todo: allow single quotes? + def splitRegex = java.util.regex.Pattern.compile(''',(?=(?:[^"]*"[^"]*")*[^"]*$)''') + def removeQuote = java.util.regex.Pattern.compile('''"(.*)"''') + + def br = java.nio.file.Files.newBufferedReader(inputFile) + + def row = -1 + def header = null + while (br.ready() && header == null) { + def line = br.readLine() + row++ + if (!line.startsWith("#")) { + header = splitRegex.split(line, -1).collect{field -> + m = removeQuote.matcher(field) + m.find() ? m.replaceFirst('$1') : field + } + } + } + assert header != null: "CSV file should contain a header" + + while (br.ready()) { + def line = br.readLine() + row++ + if (line == null) { + br.close() + break + } + + if (!line.startsWith("#")) { + def predata = splitRegex.split(line, -1) + def data = predata.collect{field -> + if (field == "") { + return null + } + def m = removeQuote.matcher(field) + if (m.find()) { + return m.replaceFirst('$1') + } else { + return field + } + } + assert header.size() == data.size(): "Row $row should contain the same number as fields as the header" + + def dataMap = [header, data].transpose().collectEntries().findAll{it.value != null} + output.add(dataMap) + } + } + + output +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readJson.nf' +def readJson(file_path) { + def inputFile = file_path !instanceof Path ? file(file_path, hidden: true) : file_path + def jsonSlurper = new groovy.json.JsonSlurper() + jsonSlurper.parse(inputFile) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readJsonBlob.nf' +def readJsonBlob(str) { + def jsonSlurper = new groovy.json.JsonSlurper() + jsonSlurper.parseText(str) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readTaggedYaml.nf' +// Custom constructor to modify how certain objects are parsed from YAML +class CustomConstructor extends org.yaml.snakeyaml.constructor.Constructor { + Path root + + class ConstructPath extends org.yaml.snakeyaml.constructor.AbstractConstruct { + public Object construct(org.yaml.snakeyaml.nodes.Node node) { + String filename = (String) constructScalar(node); + if (root != null) { + return root.resolve(filename); + } + return java.nio.file.Paths.get(filename); + } + } + + CustomConstructor(org.yaml.snakeyaml.LoaderOptions options, Path root) { + super(options) + this.root = root + // Handling !file tag and parse it back to a File type + this.yamlConstructors.put(new org.yaml.snakeyaml.nodes.Tag("!file"), new ConstructPath()) + } +} + +def readTaggedYaml(Path path) { + def options = new org.yaml.snakeyaml.LoaderOptions() + def constructor = new CustomConstructor(options, path.getParent()) + def yaml = new org.yaml.snakeyaml.Yaml(constructor) + return yaml.load(path.text) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readYaml.nf' +def readYaml(file_path) { + def inputFile = file_path !instanceof Path ? file(file_path, hidden: true) : file_path + def yamlSlurper = new org.yaml.snakeyaml.Yaml() + yamlSlurper.load(inputFile) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readYamlBlob.nf' +def readYamlBlob(str) { + def yamlSlurper = new org.yaml.snakeyaml.Yaml() + yamlSlurper.load(str) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/toJsonBlob.nf' +String toJsonBlob(data) { + return groovy.json.JsonOutput.toJson(data) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/toTaggedYamlBlob.nf' +// Custom representer to modify how certain objects are represented in YAML +class CustomRepresenter extends org.yaml.snakeyaml.representer.Representer { + Path relativizer + + class RepresentPath implements org.yaml.snakeyaml.representer.Represent { + public String getFileName(Object obj) { + if (obj instanceof File) { + obj = ((File) obj).toPath(); + } + if (obj !instanceof Path) { + throw new IllegalArgumentException("Object: " + obj + " is not a Path or File"); + } + def path = (Path) obj; + + if (relativizer != null) { + return relativizer.relativize(path).toString() + } else { + return path.toString() + } + } + + public org.yaml.snakeyaml.nodes.Node representData(Object data) { + String filename = getFileName(data); + def tag = new org.yaml.snakeyaml.nodes.Tag("!file"); + return representScalar(tag, filename); + } + } + CustomRepresenter(org.yaml.snakeyaml.DumperOptions options, Path relativizer) { + super(options) + this.relativizer = relativizer + this.representers.put(sun.nio.fs.UnixPath, new RepresentPath()) + this.representers.put(Path, new RepresentPath()) + this.representers.put(File, new RepresentPath()) + } +} + +String toTaggedYamlBlob(data) { + return toRelativeTaggedYamlBlob(data, null) +} +String toRelativeTaggedYamlBlob(data, Path relativizer) { + def options = new org.yaml.snakeyaml.DumperOptions() + options.setDefaultFlowStyle(org.yaml.snakeyaml.DumperOptions.FlowStyle.BLOCK) + def representer = new CustomRepresenter(options, relativizer) + def yaml = new org.yaml.snakeyaml.Yaml(representer, options) + return yaml.dump(data) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/toYamlBlob.nf' +String toYamlBlob(data) { + def options = new org.yaml.snakeyaml.DumperOptions() + options.setDefaultFlowStyle(org.yaml.snakeyaml.DumperOptions.FlowStyle.BLOCK) + options.setPrettyFlow(true) + def yaml = new org.yaml.snakeyaml.Yaml(options) + def cleanData = iterateMap(data, { it instanceof Path ? it.toString() : it }) + return yaml.dump(cleanData) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/writeJson.nf' +void writeJson(data, file) { + assert data: "writeJson: data should not be null" + assert file: "writeJson: file should not be null" + file.write(toJsonBlob(data)) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/writeYaml.nf' +void writeYaml(data, file) { + assert data: "writeYaml: data should not be null" + assert file: "writeYaml: file should not be null" + file.write(toYamlBlob(data)) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/findStates.nf' +def findStates(Map params, Map config) { + def auto_config = deepClone(config) + def auto_params = deepClone(params) + + auto_config = auto_config.clone() + // override arguments + auto_config.argument_groups = [] + auto_config.arguments = [ + [ + type: "string", + name: "--id", + description: "A dummy identifier", + required: false + ], + [ + type: "file", + name: "--input_states", + example: "/path/to/input/directory/**/state.yaml", + description: "Path to input directory containing the datasets to be integrated.", + required: true, + multiple: true, + multiple_sep: ";" + ], + [ + type: "string", + name: "--filter", + example: "foo/.*/state.yaml", + description: "Regex to filter state files by path.", + required: false + ], + // to do: make this a yaml blob? + [ + type: "string", + name: "--rename_keys", + example: ["newKey1:oldKey1", "newKey2:oldKey2"], + description: "Rename keys in the detected input files. This is useful if the input files do not match the set of input arguments of the workflow.", + required: false, + multiple: true, + multiple_sep: ";" + ], + [ + type: "string", + name: "--settings", + example: '{"output_dataset": "dataset.h5ad", "k": 10}', + description: "Global arguments as a JSON glob to be passed to all components.", + required: false + ] + ] + if (!(auto_params.containsKey("id"))) { + auto_params["id"] = "auto" + } + + // run auto config through processConfig once more + auto_config = processConfig(auto_config) + + workflow findStatesWf { + helpMessage(auto_config) + + output_ch = + channelFromParams(auto_params, auto_config) + | flatMap { autoId, args -> + + def globalSettings = args.settings ? readYamlBlob(args.settings) : [:] + + // look for state files in input dir + def stateFiles = args.input_states + + // filter state files by regex + if (args.filter) { + stateFiles = stateFiles.findAll{ stateFile -> + def stateFileStr = stateFile.toString() + def matcher = stateFileStr =~ args.filter + matcher.matches()} + } + + // read in states + def states = stateFiles.collect { stateFile -> + def state_ = readTaggedYaml(stateFile) + [state_.id, state_] + } + + // construct renameMap + if (args.rename_keys) { + def renameMap = args.rename_keys.collectEntries{renameString -> + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" + split + } + + // rename keys in state, only let states through which have all keys + // also add global settings + states = states.collectMany{id, state -> + def newState = [:] + + for (key in renameMap.keySet()) { + def origKey = renameMap[key] + if (!(state.containsKey(origKey))) { + return [] + } + newState[key] = state[origKey] + } + + [[id, globalSettings + newState]] + } + } + + states + } + emit: + output_ch + } + + return findStatesWf +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/joinStates.nf' +def joinStates(Closure apply_) { + workflow joinStatesWf { + take: input_ch + main: + output_ch = input_ch + | toSortedList + | filter{ it.size() > 0 } + | map{ tups -> + def ids = tups.collect{it[0]} + def states = tups.collect{it[1]} + apply_(ids, states) + } + + emit: output_ch + } + return joinStatesWf +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishStates.nf' +def collectFiles(obj) { + if (obj instanceof java.io.File || obj instanceof Path) { + return [obj] + } else if (obj instanceof List && obj !instanceof String) { + return obj.collectMany{item -> + collectFiles(item) + } + } else if (obj instanceof Map) { + return obj.collectMany{key, item -> + collectFiles(item) + } + } else { + return [] + } +} + +/** + * Recurse through a state and collect all input files and their target output filenames. + * @param obj The state to recurse through. + * @param prefix The prefix to prepend to the output filenames. + */ +def collectInputOutputPaths(obj, prefix) { + if (obj instanceof File || obj instanceof Path) { + def path = obj instanceof Path ? obj : obj.toPath() + def ext = path.getFileName().toString().find("\\.[^\\.]+\$") ?: "" + def newFilename = prefix + ext + return [[obj, newFilename]] + } else if (obj instanceof List && obj !instanceof String) { + return obj.withIndex().collectMany{item, ix -> + collectInputOutputPaths(item, prefix + "_" + ix) + } + } else if (obj instanceof Map) { + return obj.collectMany{key, item -> + collectInputOutputPaths(item, prefix + "." + key) + } + } else { + return [] + } +} + +def publishStates(Map args) { + def key_ = args.get("key") + def yamlTemplate_ = args.get("output_state", args.get("outputState", '$id.$key.state.yaml')) + + assert key_ != null : "publishStates: key must be specified" + + workflow publishStatesWf { + take: input_ch + main: + input_ch + | map { tup -> + def id_ = tup[0] + def state_ = tup[1] + + // the input files and the target output filenames + def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose() + def inputFiles_ = inputoutputFilenames_[0] + def outputFilenames_ = inputoutputFilenames_[1] + + def yamlFilename = yamlTemplate_ + .replaceAll('\\$id', id_) + .replaceAll('\\$key', key_) + + // TODO: do the pathnames in state_ match up with the outputFilenames_? + + // convert state to yaml blob + def yamlBlob_ = toRelativeTaggedYamlBlob([id: id_] + state_, java.nio.file.Paths.get(yamlFilename)) + + [id_, yamlBlob_, yamlFilename, inputFiles_, outputFilenames_] + } + | publishStatesProc + emit: input_ch + } + return publishStatesWf +} +process publishStatesProc { + // todo: check publishpath? + publishDir path: "${getPublishDir()}/", mode: "copy" + tag "$id" + input: + tuple val(id), val(yamlBlob), val(yamlFile), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles) + output: + tuple val(id), path{[yamlFile] + outputFiles} + script: + def copyCommands = [ + inputFiles instanceof List ? inputFiles : [inputFiles], + outputFiles instanceof List ? outputFiles : [outputFiles] + ] + .transpose() + .collectMany{infile, outfile -> + if (infile.toString() != outfile.toString()) { + [ + "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"", + "cp -r '${infile.toString()}' '${outfile.toString()}'" + ] + } else { + // no need to copy if infile is the same as outfile + [] + } + } + """ +mkdir -p "\$(dirname '${yamlFile}')" +echo "Storing state as yaml" +echo '${yamlBlob}' > '${yamlFile}' +echo "Copying output files to destination folder" +${copyCommands.join("\n ")} +""" +} + + +// this assumes that the state contains no other values other than those specified in the config +def publishStatesByConfig(Map args) { + def config = args.get("config") + assert config != null : "publishStatesByConfig: config must be specified" + + def key_ = args.get("key", config.name) + assert key_ != null : "publishStatesByConfig: key must be specified" + + workflow publishStatesSimpleWf { + take: input_ch + main: + input_ch + | map { tup -> + def id_ = tup[0] + def state_ = tup[1] // e.g. [output: new File("myoutput.h5ad"), k: 10] + def origState_ = tup[2] // e.g. [output: '$id.$key.foo.h5ad'] + + // TODO: allow overriding the state.yaml template + // TODO TODO: if auto.publish == "state", add output_state as an argument + def yamlTemplate = params.containsKey("output_state") ? params.output_state : '$id.$key.state.yaml' + def yamlFilename = yamlTemplate + .replaceAll('\\$id', id_) + .replaceAll('\\$key', key_) + def yamlDir = java.nio.file.Paths.get(yamlFilename).getParent() + + // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where + // - key is a String + // - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path) + // - inputPath is a List[Path] + // - outputFilename is a List[String] + // - (key, value) are the tuples that will be saved to the state.yaml file + // - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml) + def processedState = + config.allArguments + .findAll { it.direction == "output" } + .collectMany { par -> + def plainName_ = par.plainName + // if the state does not contain the key, it's an + // optional argument for which the component did + // not generate any output + if (!state_.containsKey(plainName_)) { + return [] + } + def value = state_[plainName_] + // if the parameter is not a file, it should be stored + // in the state as-is, but is not something that needs + // to be copied from the source path to the dest path + if (par.type != "file") { + return [[key: plainName_, value: value, inputPath: [], outputFilename: []]] + } + // if the orig state does not contain this filename, + // it's an optional argument for which the user specified + // that it should not be returned as a state + if (!origState_.containsKey(plainName_)) { + return [] + } + def filenameTemplate = origState_[plainName_] + // if the pararameter is multiple: true, fetch the template + if (par.multiple && filenameTemplate instanceof List) { + filenameTemplate = filenameTemplate[0] + } + // instantiate the template + def filename = filenameTemplate + .replaceAll('\\$id', id_) + .replaceAll('\\$key', key_) + if (par.multiple) { + // if the parameter is multiple: true, the filename + // should contain a wildcard '*' that is replaced with + // the index of the file + assert filename.contains("*") : "Module '${key_}' id '${id_}': Multiple output files specified, but no wildcard '*' in the filename: ${filename}" + def outputPerFile = value.withIndex().collect{ val, ix -> + def filename_ix = filename.replace("*", ix.toString()) + def value_ = java.nio.file.Paths.get(filename_ix) + // if id contains a slash + if (yamlDir != null) { + value_ = yamlDir.relativize(value_) + } + def inputPath = val instanceof File ? val.toPath() : val + [value: value_, inputPath: inputPath, outputFilename: filename_ix] + } + def transposedOutputs = ["value", "inputPath", "outputFilename"].collectEntries{ key -> + [key, outputPerFile.collect{dic -> dic[key]}] + } + return [[key: plainName_] + transposedOutputs] + } else { + def value_ = java.nio.file.Paths.get(filename) + // if id contains a slash + if (yamlDir != null) { + value_ = yamlDir.relativize(value_) + } + def inputPath = value instanceof File ? value.toPath() : value + return [[key: plainName_, value: value_, inputPath: [inputPath], outputFilename: [filename]]] + } + } + + def updatedState_ = processedState.collectEntries{[it.key, it.value]} + def inputPaths = processedState.collectMany{it.inputPath} + def outputFilenames = processedState.collectMany{it.outputFilename} + + // convert state to yaml blob + def yamlBlob_ = toTaggedYamlBlob([id: id_] + updatedState_) + + [id_, yamlBlob_, yamlFilename, inputPaths, outputFilenames] + } + | publishStatesProc + emit: input_ch + } + return publishStatesSimpleWf +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/setState.nf' +def setState(fun) { + assert fun instanceof Closure || fun instanceof Map || fun instanceof List : + "Error in setState: Expected process argument to be a Closure, a Map, or a List. Found: class ${fun.getClass()}" + + // if fun is a List, convert to map + if (fun instanceof List) { + // check whether fun is a list[string] + assert fun.every{it instanceof CharSequence} : "Error in setState: argument is a List, but not all elements are Strings" + fun = fun.collectEntries{[it, it]} + } + + // if fun is a map, convert to closure + if (fun instanceof Map) { + // check whether fun is a map[string, string] + assert fun.values().every{it instanceof CharSequence} : "Error in setState: argument is a Map, but not all values are Strings" + assert fun.keySet().every{it instanceof CharSequence} : "Error in setState: argument is a Map, but not all keys are Strings" + def funMap = fun.clone() + // turn the map into a closure to be used later on + fun = { id_, state_ -> + assert state_ instanceof Map : "Error in setState: the state is not a Map" + funMap.collectMany{newkey, origkey -> + if (state_.containsKey(origkey)) { + [[newkey, state_[origkey]]] + } else { + [] + } + }.collectEntries() + } + } + + map { tup -> + def id = tup[0] + def state = tup[1] + def unfilteredState = fun(id, state) + def newState = unfilteredState.findAll{key, val -> val != null} + [id, newState] + tup.drop(2) + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/processAuto.nf' +// TODO: unit test processAuto +def processAuto(Map auto) { + // remove null values + auto = auto.findAll{k, v -> v != null} + + // check for unexpected keys + def expectedKeys = ["simplifyInput", "simplifyOutput", "transcript", "publish"] + def unexpectedKeys = auto.keySet() - expectedKeys + assert unexpectedKeys.isEmpty(), "unexpected keys in auto: '${unexpectedKeys.join("', '")}'" + + // check auto.simplifyInput + assert auto.simplifyInput instanceof Boolean, "auto.simplifyInput must be a boolean" + + // check auto.simplifyOutput + assert auto.simplifyOutput instanceof Boolean, "auto.simplifyOutput must be a boolean" + + // check auto.transcript + assert auto.transcript instanceof Boolean, "auto.transcript must be a boolean" + + // check auto.publish + assert auto.publish instanceof Boolean || auto.publish == "state", "auto.publish must be a boolean or 'state'" + + return auto.subMap(expectedKeys) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/processDirectives.nf' +def assertMapKeys(map, expectedKeys, requiredKeys, mapName) { + assert map instanceof Map : "Expected argument '$mapName' to be a Map. Found: class ${map.getClass()}" + map.forEach { key, val -> + assert key in expectedKeys : "Unexpected key '$key' in ${mapName ? mapName + " " : ""}map" + } + requiredKeys.forEach { requiredKey -> + assert map.containsKey(requiredKey) : "Missing required key '$key' in ${mapName ? mapName + " " : ""}map" + } +} + +// TODO: unit test processDirectives +def processDirectives(Map drctv) { + // remove null values + drctv = drctv.findAll{k, v -> v != null} + + // check for unexpected keys + def expectedKeys = [ + "accelerator", "afterScript", "beforeScript", "cache", "conda", "container", "containerOptions", "cpus", "disk", "echo", "errorStrategy", "executor", "machineType", "maxErrors", "maxForks", "maxRetries", "memory", "module", "penv", "pod", "publishDir", "queue", "label", "scratch", "storeDir", "stageInMode", "stageOutMode", "tag", "time" + ] + def unexpectedKeys = drctv.keySet() - expectedKeys + assert unexpectedKeys.isEmpty() : "Unexpected keys in process directive: '${unexpectedKeys.join("', '")}'" + + /* DIRECTIVE accelerator + accepted examples: + - [ limit: 4, type: "nvidia-tesla-k80" ] + */ + if (drctv.containsKey("accelerator")) { + assertMapKeys(drctv["accelerator"], ["type", "limit", "request", "runtime"], [], "accelerator") + } + + /* DIRECTIVE afterScript + accepted examples: + - "source /cluster/bin/cleanup" + */ + if (drctv.containsKey("afterScript")) { + assert drctv["afterScript"] instanceof CharSequence + } + + /* DIRECTIVE beforeScript + accepted examples: + - "source /cluster/bin/setup" + */ + if (drctv.containsKey("beforeScript")) { + assert drctv["beforeScript"] instanceof CharSequence + } + + /* DIRECTIVE cache + accepted examples: + - true + - false + - "deep" + - "lenient" + */ + if (drctv.containsKey("cache")) { + assert drctv["cache"] instanceof CharSequence || drctv["cache"] instanceof Boolean + if (drctv["cache"] instanceof CharSequence) { + assert drctv["cache"] in ["deep", "lenient"] : "Unexpected value for cache" + } + } + + /* DIRECTIVE conda + accepted examples: + - "bwa=0.7.15" + - "bwa=0.7.15 fastqc=0.11.5" + - ["bwa=0.7.15", "fastqc=0.11.5"] + */ + if (drctv.containsKey("conda")) { + if (drctv["conda"] instanceof List) { + drctv["conda"] = drctv["conda"].join(" ") + } + assert drctv["conda"] instanceof CharSequence + } + + /* DIRECTIVE container + accepted examples: + - "foo/bar:tag" + - [ registry: "reg", image: "im", tag: "ta" ] + is transformed to "reg/im:ta" + - [ image: "im" ] + is transformed to "im:latest" + */ + if (drctv.containsKey("container")) { + assert drctv["container"] instanceof Map || drctv["container"] instanceof CharSequence + if (drctv["container"] instanceof Map) { + def m = drctv["container"] + assertMapKeys(m, [ "registry", "image", "tag" ], ["image"], "container") + def part1 = + System.getenv('OVERRIDE_CONTAINER_REGISTRY') ? System.getenv('OVERRIDE_CONTAINER_REGISTRY') + "/" : + params.containsKey("override_container_registry") ? params["override_container_registry"] + "/" : // todo: remove? + m.registry ? m.registry + "/" : + "" + def part2 = m.image + def part3 = m.tag ? ":" + m.tag : ":latest" + drctv["container"] = part1 + part2 + part3 + } + } + + /* DIRECTIVE containerOptions + accepted examples: + - "--foo bar" + - ["--foo bar", "-f b"] + */ + if (drctv.containsKey("containerOptions")) { + if (drctv["containerOptions"] instanceof List) { + drctv["containerOptions"] = drctv["containerOptions"].join(" ") + } + assert drctv["containerOptions"] instanceof CharSequence + } + + /* DIRECTIVE cpus + accepted examples: + - 1 + - 10 + */ + if (drctv.containsKey("cpus")) { + assert drctv["cpus"] instanceof Integer + } + + /* DIRECTIVE disk + accepted examples: + - "1 GB" + - "2TB" + - "3.2KB" + - "10.B" + */ + if (drctv.containsKey("disk")) { + assert drctv["disk"] instanceof CharSequence + // assert drctv["disk"].matches("[0-9]+(\\.[0-9]*)? *[KMGTPEZY]?B") + // ^ does not allow closures + } + + /* DIRECTIVE echo + accepted examples: + - true + - false + */ + if (drctv.containsKey("echo")) { + assert drctv["echo"] instanceof Boolean + } + + /* DIRECTIVE errorStrategy + accepted examples: + - "terminate" + - "finish" + */ + if (drctv.containsKey("errorStrategy")) { + assert drctv["errorStrategy"] instanceof CharSequence + assert drctv["errorStrategy"] in ["terminate", "finish", "ignore", "retry"] : "Unexpected value for errorStrategy" + } + + /* DIRECTIVE executor + accepted examples: + - "local" + - "sge" + */ + if (drctv.containsKey("executor")) { + assert drctv["executor"] instanceof CharSequence + assert drctv["executor"] in ["local", "sge", "uge", "lsf", "slurm", "pbs", "pbspro", "moab", "condor", "nqsii", "ignite", "k8s", "awsbatch", "google-pipelines"] : "Unexpected value for executor" + } + + /* DIRECTIVE machineType + accepted examples: + - "n1-highmem-8" + */ + if (drctv.containsKey("machineType")) { + assert drctv["machineType"] instanceof CharSequence + } + + /* DIRECTIVE maxErrors + accepted examples: + - 1 + - 3 + */ + if (drctv.containsKey("maxErrors")) { + assert drctv["maxErrors"] instanceof Integer + } + + /* DIRECTIVE maxForks + accepted examples: + - 1 + - 3 + */ + if (drctv.containsKey("maxForks")) { + assert drctv["maxForks"] instanceof Integer + } + + /* DIRECTIVE maxRetries + accepted examples: + - 1 + - 3 + */ + if (drctv.containsKey("maxRetries")) { + assert drctv["maxRetries"] instanceof Integer + } + + /* DIRECTIVE memory + accepted examples: + - "1 GB" + - "2TB" + - "3.2KB" + - "10.B" + */ + if (drctv.containsKey("memory")) { + assert drctv["memory"] instanceof CharSequence + // assert drctv["memory"].matches("[0-9]+(\\.[0-9]*)? *[KMGTPEZY]?B") + // ^ does not allow closures + } + + /* DIRECTIVE module + accepted examples: + - "ncbi-blast/2.2.27" + - "ncbi-blast/2.2.27:t_coffee/10.0" + - ["ncbi-blast/2.2.27", "t_coffee/10.0"] + */ + if (drctv.containsKey("module")) { + if (drctv["module"] instanceof List) { + drctv["module"] = drctv["module"].join(":") + } + assert drctv["module"] instanceof CharSequence + } + + /* DIRECTIVE penv + accepted examples: + - "smp" + */ + if (drctv.containsKey("penv")) { + assert drctv["penv"] instanceof CharSequence + } + + /* DIRECTIVE pod + accepted examples: + - [ label: "key", value: "val" ] + - [ annotation: "key", value: "val" ] + - [ env: "key", value: "val" ] + - [ [label: "l", value: "v"], [env: "e", value: "v"]] + */ + if (drctv.containsKey("pod")) { + if (drctv["pod"] instanceof Map) { + drctv["pod"] = [ drctv["pod"] ] + } + assert drctv["pod"] instanceof List + drctv["pod"].forEach { pod -> + assert pod instanceof Map + // TODO: should more checks be added? + // See https://www.nextflow.io/docs/latest/process.html?highlight=directives#pod + // e.g. does it contain 'label' and 'value', or 'annotation' and 'value', or ...? + } + } + + /* DIRECTIVE publishDir + accepted examples: + - [] + - [ [ path: "foo", enabled: true ], [ path: "bar", enabled: false ] ] + - "/path/to/dir" + is transformed to [[ path: "/path/to/dir" ]] + - [ path: "/path/to/dir", mode: "cache" ] + is transformed to [[ path: "/path/to/dir", mode: "cache" ]] + */ + // TODO: should we also look at params["publishDir"]? + if (drctv.containsKey("publishDir")) { + def pblsh = drctv["publishDir"] + + // check different options + assert pblsh instanceof List || pblsh instanceof Map || pblsh instanceof CharSequence + + // turn into list if not already so + // for some reason, 'if (!pblsh instanceof List) pblsh = [ pblsh ]' doesn't work. + pblsh = pblsh instanceof List ? pblsh : [ pblsh ] + + // check elements of publishDir + pblsh = pblsh.collect{ elem -> + // turn into map if not already so + elem = elem instanceof CharSequence ? [ path: elem ] : elem + + // check types and keys + assert elem instanceof Map : "Expected publish argument '$elem' to be a String or a Map. Found: class ${elem.getClass()}" + assertMapKeys(elem, [ "path", "mode", "overwrite", "pattern", "saveAs", "enabled" ], ["path"], "publishDir") + + // check elements in map + assert elem.containsKey("path") + assert elem["path"] instanceof CharSequence + if (elem.containsKey("mode")) { + assert elem["mode"] instanceof CharSequence + assert elem["mode"] in [ "symlink", "rellink", "link", "copy", "copyNoFollow", "move" ] + } + if (elem.containsKey("overwrite")) { + assert elem["overwrite"] instanceof Boolean + } + if (elem.containsKey("pattern")) { + assert elem["pattern"] instanceof CharSequence + } + if (elem.containsKey("saveAs")) { + assert elem["saveAs"] instanceof CharSequence //: "saveAs as a Closure is currently not supported. Surround your closure with single quotes to get the desired effect. Example: '\{ foo \}'" + } + if (elem.containsKey("enabled")) { + assert elem["enabled"] instanceof Boolean + } + + // return final result + elem + } + // store final directive + drctv["publishDir"] = pblsh + } + + /* DIRECTIVE queue + accepted examples: + - "long" + - "short,long" + - ["short", "long"] + */ + if (drctv.containsKey("queue")) { + if (drctv["queue"] instanceof List) { + drctv["queue"] = drctv["queue"].join(",") + } + assert drctv["queue"] instanceof CharSequence + } + + /* DIRECTIVE label + accepted examples: + - "big_mem" + - "big_cpu" + - ["big_mem", "big_cpu"] + */ + if (drctv.containsKey("label")) { + if (drctv["label"] instanceof CharSequence) { + drctv["label"] = [ drctv["label"] ] + } + assert drctv["label"] instanceof List + drctv["label"].forEach { label -> + assert label instanceof CharSequence + // assert label.matches("[a-zA-Z0-9]([a-zA-Z0-9_]*[a-zA-Z0-9])?") + // ^ does not allow closures + } + } + + /* DIRECTIVE scratch + accepted examples: + - true + - "/path/to/scratch" + - '$MY_PATH_TO_SCRATCH' + - "ram-disk" + */ + if (drctv.containsKey("scratch")) { + assert drctv["scratch"] == true || drctv["scratch"] instanceof CharSequence + } + + /* DIRECTIVE storeDir + accepted examples: + - "/path/to/storeDir" + */ + if (drctv.containsKey("storeDir")) { + assert drctv["storeDir"] instanceof CharSequence + } + + /* DIRECTIVE stageInMode + accepted examples: + - "copy" + - "link" + */ + if (drctv.containsKey("stageInMode")) { + assert drctv["stageInMode"] instanceof CharSequence + assert drctv["stageInMode"] in ["copy", "link", "symlink", "rellink"] + } + + /* DIRECTIVE stageOutMode + accepted examples: + - "copy" + - "link" + */ + if (drctv.containsKey("stageOutMode")) { + assert drctv["stageOutMode"] instanceof CharSequence + assert drctv["stageOutMode"] in ["copy", "move", "rsync"] + } + + /* DIRECTIVE tag + accepted examples: + - "foo" + - '$id' + */ + if (drctv.containsKey("tag")) { + assert drctv["tag"] instanceof CharSequence + } + + /* DIRECTIVE time + accepted examples: + - "1h" + - "2days" + - "1day 6hours 3minutes 30seconds" + */ + if (drctv.containsKey("time")) { + assert drctv["time"] instanceof CharSequence + // todo: validation regex? + } + + return drctv +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/processWorkflowArgs.nf' +def processWorkflowArgs(Map args, Map defaultWfArgs, Map meta) { + // override defaults with args + def workflowArgs = defaultWfArgs + args + + // check whether 'key' exists + assert workflowArgs.containsKey("key") : "Error in module '${meta.config.name}': key is a required argument" + + // if 'key' is a closure, apply it to the original key + if (workflowArgs["key"] instanceof Closure) { + workflowArgs["key"] = workflowArgs["key"](meta.config.name) + } + def key = workflowArgs["key"] + assert key instanceof CharSequence : "Expected process argument 'key' to be a String. Found: class ${key.getClass()}" + assert key ==~ /^[a-zA-Z_]\w*$/ : "Error in module '$key': Expected process argument 'key' to consist of only letters, digits or underscores. Found: ${key}" + + // check for any unexpected keys + def expectedKeys = ["key", "directives", "auto", "map", "mapId", "mapData", "mapPassthrough", "filter", "runIf", "fromState", "toState", "args", "renameKeys", "debug"] + def unexpectedKeys = workflowArgs.keySet() - expectedKeys + assert unexpectedKeys.isEmpty() : "Error in module '$key': unexpected arguments to the '.run()' function: '${unexpectedKeys.join("', '")}'" + + // check whether directives exists and apply defaults + assert workflowArgs.containsKey("directives") : "Error in module '$key': directives is a required argument" + assert workflowArgs["directives"] instanceof Map : "Error in module '$key': Expected process argument 'directives' to be a Map. Found: class ${workflowArgs['directives'].getClass()}" + workflowArgs["directives"] = processDirectives(defaultWfArgs.directives + workflowArgs["directives"]) + + // check whether directives exists and apply defaults + assert workflowArgs.containsKey("auto") : "Error in module '$key': auto is a required argument" + assert workflowArgs["auto"] instanceof Map : "Error in module '$key': Expected process argument 'auto' to be a Map. Found: class ${workflowArgs['auto'].getClass()}" + workflowArgs["auto"] = processAuto(defaultWfArgs.auto + workflowArgs["auto"]) + + // auto define publish, if so desired + if (workflowArgs.auto.publish == true && (workflowArgs.directives.publishDir != null ? workflowArgs.directives.publishDir : [:]).isEmpty()) { + // can't assert at this level thanks to the no_publish profile + // assert params.containsKey("publishDir") || params.containsKey("publish_dir") : + // "Error in module '${workflowArgs['key']}': if auto.publish is true, params.publish_dir needs to be defined.\n" + + // " Example: params.publish_dir = \"./output/\"" + def publishDir = getPublishDir() + + if (publishDir != null) { + workflowArgs.directives.publishDir = [[ + path: publishDir, + saveAs: "{ it.startsWith('.') ? null : it }", // don't publish hidden files, by default + mode: "copy" + ]] + } + } + + // auto define transcript, if so desired + if (workflowArgs.auto.transcript == true) { + // can't assert at this level thanks to the no_publish profile + // assert params.containsKey("transcriptsDir") || params.containsKey("transcripts_dir") || params.containsKey("publishDir") || params.containsKey("publish_dir") : + // "Error in module '${workflowArgs['key']}': if auto.transcript is true, either params.transcripts_dir or params.publish_dir needs to be defined.\n" + + // " Example: params.transcripts_dir = \"./transcripts/\"" + def transcriptsDir = + params.containsKey("transcripts_dir") ? params.transcripts_dir : + params.containsKey("transcriptsDir") ? params.transcriptsDir : + params.containsKey("publish_dir") ? params.publish_dir + "/_transcripts" : + params.containsKey("publishDir") ? params.publishDir + "/_transcripts" : + null + if (transcriptsDir != null) { + def timestamp = nextflow.Nextflow.getSession().getWorkflowMetadata().start.format('yyyy-MM-dd_HH-mm-ss') + def transcriptsPublishDir = [ + path: "$transcriptsDir/$timestamp/\${task.process.replaceAll(':', '-')}/\${id}/", + saveAs: "{ it.startsWith('.') ? it.replaceAll('^.', '') : null }", + mode: "copy" + ] + def publishDirs = workflowArgs.directives.publishDir != null ? workflowArgs.directives.publishDir : null ? workflowArgs.directives.publishDir : [] + workflowArgs.directives.publishDir = publishDirs + transcriptsPublishDir + } + } + + // if this is a stubrun, remove certain directives? + if (workflow.stubRun) { + workflowArgs.directives.keySet().removeAll(["publishDir", "cpus", "memory", "label"]) + } + + for (nam in ["map", "mapId", "mapData", "mapPassthrough", "filter", "runIf"]) { + if (workflowArgs.containsKey(nam) && workflowArgs[nam]) { + assert workflowArgs[nam] instanceof Closure : "Error in module '$key': Expected process argument '$nam' to be null or a Closure. Found: class ${workflowArgs[nam].getClass()}" + } + } + + // TODO: should functions like 'map', 'mapId', 'mapData', 'mapPassthrough' be deprecated as well? + for (nam in ["map", "mapData", "mapPassthrough", "renameKeys"]) { + if (workflowArgs.containsKey(nam) && workflowArgs[nam] != null) { + log.warn "module '$key': workflow argument '$nam' is deprecated and will be removed in Viash 0.9.0. Please use 'fromState' and 'toState' instead." + } + } + + // check fromState + workflowArgs["fromState"] = _processFromState(workflowArgs.get("fromState"), key, meta.config) + + // check toState + workflowArgs["toState"] = _processToState(workflowArgs.get("toState"), key, meta.config) + + // return output + return workflowArgs +} + +def _processFromState(fromState, key_, config_) { + assert fromState == null || fromState instanceof Closure || fromState instanceof Map || fromState instanceof List : + "Error in module '$key_': Expected process argument 'fromState' to be null, a Closure, a Map, or a List. Found: class ${fromState.getClass()}" + if (fromState == null) { + return null + } + + // if fromState is a List, convert to map + if (fromState instanceof List) { + // check whether fromstate is a list[string] + assert fromState.every{it instanceof CharSequence} : "Error in module '$key_': fromState is a List, but not all elements are Strings" + fromState = fromState.collectEntries{[it, it]} + } + + // if fromState is a map, convert to closure + if (fromState instanceof Map) { + // check whether fromstate is a map[string, string] + assert fromState.values().every{it instanceof CharSequence} : "Error in module '$key_': fromState is a Map, but not all values are Strings" + assert fromState.keySet().every{it instanceof CharSequence} : "Error in module '$key_': fromState is a Map, but not all keys are Strings" + def fromStateMap = fromState.clone() + def requiredInputNames = meta.config.allArguments.findAll{it.required && it.direction == "Input"}.collect{it.plainName} + // turn the map into a closure to be used later on + fromState = { it -> + def state = it[1] + assert state instanceof Map : "Error in module '$key_': the state is not a Map" + def data = fromStateMap.collectMany{newkey, origkey -> + // check whether newkey corresponds to a required argument + if (state.containsKey(origkey)) { + [[newkey, state[origkey]]] + } else if (!requiredInputNames.contains(origkey)) { + [] + } else { + throw new Exception("Error in module '$key_': fromState key '$origkey' not found in current state") + } + }.collectEntries() + data + } + } + + return fromState +} + +def _processToState(toState, key_, config_) { + if (toState == null) { + toState = { tup -> tup[1] } + } + + // toState should be a closure, map[string, string], or list[string] + assert toState instanceof Closure || toState instanceof Map || toState instanceof List : + "Error in module '$key_': Expected process argument 'toState' to be a Closure, a Map, or a List. Found: class ${toState.getClass()}" + + // if toState is a List, convert to map + if (toState instanceof List) { + // check whether toState is a list[string] + assert toState.every{it instanceof CharSequence} : "Error in module '$key_': toState is a List, but not all elements are Strings" + toState = toState.collectEntries{[it, it]} + } + + // if toState is a map, convert to closure + if (toState instanceof Map) { + // check whether toState is a map[string, string] + assert toState.values().every{it instanceof CharSequence} : "Error in module '$key_': toState is a Map, but not all values are Strings" + assert toState.keySet().every{it instanceof CharSequence} : "Error in module '$key_': toState is a Map, but not all keys are Strings" + def toStateMap = toState.clone() + def requiredOutputNames = config_.allArguments.findAll{it.required && it.direction == "Output"}.collect{it.plainName} + // turn the map into a closure to be used later on + toState = { it -> + def output = it[1] + def state = it[2] + assert output instanceof Map : "Error in module '$key_': the output is not a Map" + assert state instanceof Map : "Error in module '$key_': the state is not a Map" + def extraEntries = toStateMap.collectMany{newkey, origkey -> + // check whether newkey corresponds to a required argument + if (output.containsKey(origkey)) { + [[newkey, output[origkey]]] + } else if (!requiredOutputNames.contains(origkey)) { + [] + } else { + throw new Exception("Error in module '$key_': toState key '$origkey' not found in current output") + } + }.collectEntries() + state + extraEntries + } + } + + return toState +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/workflowFactory.nf' +def _debug(workflowArgs, debugKey) { + if (workflowArgs.debug) { + view { "process '${workflowArgs.key}' $debugKey tuple: $it" } + } else { + map { it } + } +} + +// depends on: innerWorkflowFactory +def workflowFactory(Map args, Map defaultWfArgs, Map meta) { + def workflowArgs = processWorkflowArgs(args, defaultWfArgs, meta) + def key_ = workflowArgs["key"] + + workflow workflowInstance { + take: input_ + + main: + def chModified = input_ + | checkUniqueIds([:]) + | _debug(workflowArgs, "input") + | map { tuple -> + tuple = deepClone(tuple) + + if (workflowArgs.map) { + tuple = workflowArgs.map(tuple) + } + if (workflowArgs.mapId) { + tuple[0] = workflowArgs.mapId(tuple[0]) + } + if (workflowArgs.mapData) { + tuple[1] = workflowArgs.mapData(tuple[1]) + } + if (workflowArgs.mapPassthrough) { + tuple = tuple.take(2) + workflowArgs.mapPassthrough(tuple.drop(2)) + } + + // check tuple + assert tuple instanceof List : + "Error in module '${key_}': element in channel should be a tuple [id, data, ...otherargs...]\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Expected class: List. Found: tuple.getClass() is ${tuple.getClass()}" + assert tuple.size() >= 2 : + "Error in module '${key_}': expected length of tuple in input channel to be two or greater.\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Found: tuple.size() == ${tuple.size()}" + + // check id field + if (tuple[0] instanceof GString) { + tuple[0] = tuple[0].toString() + } + assert tuple[0] instanceof CharSequence : + "Error in module '${key_}': first element of tuple in channel should be a String\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Found: ${tuple[0]}" + + // match file to input file + if (workflowArgs.auto.simplifyInput && (tuple[1] instanceof Path || tuple[1] instanceof List)) { + def inputFiles = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "input" } + + assert inputFiles.size() == 1 : + "Error in module '${key_}' id '${tuple[0]}'.\n" + + " Anonymous file inputs are only allowed when the process has exactly one file input.\n" + + " Expected: inputFiles.size() == 1. Found: inputFiles.size() is ${inputFiles.size()}" + + tuple[1] = [[ inputFiles[0].plainName, tuple[1] ]].collectEntries() + } + + // check data field + assert tuple[1] instanceof Map : + "Error in module '${key_}' id '${tuple[0]}': second element of tuple in channel should be a Map\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Expected class: Map. Found: tuple[1].getClass() is ${tuple[1].getClass()}" + + // rename keys of data field in tuple + if (workflowArgs.renameKeys) { + assert workflowArgs.renameKeys instanceof Map : + "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + + " Example: renameKeys: ['new_key': 'old_key'].\n" + + " Expected class: Map. Found: renameKeys.getClass() is ${workflowArgs.renameKeys.getClass()}" + assert tuple[1] instanceof Map : + "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + + " Expected class: Map. Found: tuple[1].getClass() is ${tuple[1].getClass()}" + + // TODO: allow renameKeys to be a function? + workflowArgs.renameKeys.each { newKey, oldKey -> + assert newKey instanceof CharSequence : + "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + + " Example: renameKeys: ['new_key': 'old_key'].\n" + + " Expected class of newKey: String. Found: newKey.getClass() is ${newKey.getClass()}" + assert oldKey instanceof CharSequence : + "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + + " Example: renameKeys: ['new_key': 'old_key'].\n" + + " Expected class of oldKey: String. Found: oldKey.getClass() is ${oldKey.getClass()}" + assert tuple[1].containsKey(oldKey) : + "Error renaming data keys in module '${key}' id '${tuple[0]}'.\n" + + " Key '$oldKey' is missing in the data map. tuple[1].keySet() is '${tuple[1].keySet()}'" + tuple[1].put(newKey, tuple[1][oldKey]) + } + tuple[1].keySet().removeAll(workflowArgs.renameKeys.collect{ newKey, oldKey -> oldKey }) + } + tuple + } + + + def chRun = null + def chPassthrough = null + if (workflowArgs.runIf) { + def runIfBranch = chModified.branch{ tup -> + run: workflowArgs.runIf(tup[0], tup[1]) + passthrough: true + } + chRun = runIfBranch.run + chPassthrough = runIfBranch.passthrough + } else { + chRun = chModified + chPassthrough = Channel.empty() + } + + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + + def chArgs = workflowArgs.fromState ? + chRunFiltered | map{ + def new_data = workflowArgs.fromState(it.take(2)) + [it[0], new_data] + } : + chRunFiltered | map {tup -> tup.take(2)} + + // fill in defaults + def chArgsWithDefaults = chArgs + | map { tuple -> + def id_ = tuple[0] + def data_ = tuple[1] + + // TODO: could move fromState to here + + // fetch default params from functionality + def defaultArgs = meta.config.allArguments + .findAll { it.containsKey("default") } + .collectEntries { [ it.plainName, it.default ] } + + // fetch overrides in params + def paramArgs = meta.config.allArguments + .findAll { par -> + def argKey = key_ + "__" + par.plainName + params.containsKey(argKey) + } + .collectEntries { [ it.plainName, params[key_ + "__" + it.plainName] ] } + + // fetch overrides in data + def dataArgs = meta.config.allArguments + .findAll { data_.containsKey(it.plainName) } + .collectEntries { [ it.plainName, data_[it.plainName] ] } + + // combine params + def combinedArgs = defaultArgs + paramArgs + workflowArgs.args + dataArgs + + // remove arguments with explicit null values + combinedArgs + .removeAll{_, val -> val == null || val == "viash_no_value" || val == "force_null"} + + combinedArgs = _processInputValues(combinedArgs, meta.config, id_, key_) + + [id_, combinedArgs] + tuple.drop(2) + } + + // TODO: move some of the _meta.join_id wrangling to the safeJoin() function. + def chInitialOutput = chArgsWithDefaults + | _debug(workflowArgs, "processed") + // run workflow + | innerWorkflowFactory(workflowArgs) + // check output tuple + | map { id_, output_ -> + + // see if output map contains metadata + def meta_ = + output_ instanceof Map && output_.containsKey("_meta") ? + output_["_meta"] : + [:] + def join_id = meta_.join_id ?: id_ + + // remove metadata + output_ = output_.findAll{k, v -> k != "_meta"} + + // check value types + output_ = _processOutputValues(output_, meta.config, id_, key_) + + // simplify output if need be + if (workflowArgs.auto.simplifyOutput && output_.size() == 1) { + output_ = output_.values()[0] + } + + [join_id, id_, output_] + } + // | view{"chInitialOutput: ${it.take(3)}"} + + // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) + // input tuple format: [join_id, id, output, prev_state, ...] + // output tuple format: [join_id, id, new_state, ...] + | map{ tup -> + def new_state = workflowArgs.toState(tup.drop(1).take(3)) + tup.take(2) + [new_state] + tup.drop(4) + } + + if (workflowArgs.auto.publish == "state") { + def chPublish = chNewState + // input tuple format: [join_id, id, new_state, ...] + // output tuple format: [join_id, id, new_state] + | map{ tup -> + tup.take(3) + } + + safeJoin(chPublish, chArgsWithDefaults, key_) + // input tuple format: [join_id, id, new_state, orig_state, ...] + // output tuple format: [id, new_state, orig_state] + | map { tup -> + tup.drop(1).take(3) + } + | publishStatesByConfig(key: key_, config: meta.config) + } + + // remove join_id and meta + chReturn = chNewState + | map { tup -> + // input tuple format: [join_id, id, new_state, ...] + // output tuple format: [id, new_state, ...] + tup.drop(1) + } + | _debug(workflowArgs, "output") + | concat(chPassthrough) + + emit: chReturn + } + + def wf = workflowInstance.cloneWithName(key_) + + // add factory function + wf.metaClass.run = { runArgs -> + workflowFactory(runArgs, workflowArgs, meta) + } + // add config to module for later introspection + wf.metaClass.config = meta.config + + return wf +} + +nextflow.enable.dsl=2 + +// START COMPONENT-SPECIFIC CODE + +// create meta object +meta = [ + "resources_dir": moduleDir.toRealPath().normalize(), + "config": processConfig(readJsonBlob('''{ + "name" : "bedtools_groupby", + "namespace" : "bedtools", + "version" : "main", + "authors" : [ + { + "name" : "Theodoro Gasperin Terra Camargo", + "roles" : [ + "author", + "maintainer" + ], + "info" : { + "links" : { + "email" : "theodorogtc@gmail.com", + "github" : "tgaspe", + "linkedin" : "theodoro-gasperin-terra-camargo" + }, + "organizations" : [ + { + "name" : "Data Intuitive", + "href" : "https://www.data-intuitive.com", + "role" : "Bioinformatician" + } + ] + } + } + ], + "argument_groups" : [ + { + "name" : "Inputs", + "arguments" : [ + { + "type" : "file", + "name" : "--input", + "alternatives" : [ + "-i" + ], + "description" : "The input BED file to be used.\n", + "example" : [ + "input_a.bed" + ], + "must_exist" : true, + "create_parent" : true, + "required" : true, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + } + ] + }, + { + "name" : "Outputs", + "arguments" : [ + { + "type" : "file", + "name" : "--output", + "description" : "The output groupby BED file. \n", + "example" : [ + "output.bed" + ], + "must_exist" : true, + "create_parent" : true, + "required" : true, + "direction" : "output", + "multiple" : false, + "multiple_sep" : ";" + } + ] + }, + { + "name" : "Options", + "arguments" : [ + { + "type" : "string", + "name" : "--groupby", + "alternatives" : [ + "-g", + "-grp" + ], + "description" : "Specify the columns (1-based) for the grouping.\nThe columns must be comma separated.\n- Default: 1,2,3 \n", + "required" : true, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + }, + { + "type" : "integer", + "name" : "--column", + "alternatives" : [ + "-c", + "-opCols" + ], + "description" : "Specify the column (1-based) that should be summarized.\n", + "required" : true, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + }, + { + "type" : "string", + "name" : "--operation", + "alternatives" : [ + "-o", + "-ops" + ], + "description" : "Specify the operation that should be applied to opCol.\nValid operations:\n sum, count, count_distinct, min, max,\n mean, median, mode, antimode,\n stdev, sstdev (sample standard dev.),\n collapse (i.e., print a comma separated list (duplicates allowed)), \n distinct (i.e., print a comma separated list (NO duplicates allowed)), \n distinct_sort_num (as distinct, but sorted numerically, ascending), \n distinct_sort_num_desc (as distinct, but sorted numerically, descending), \n concat (i.e., merge values into a single, non-delimited string), \n freqdesc (i.e., print desc. list of values:freq)\n freqasc (i.e., print asc. list of values:freq)\n first (i.e., print first value)\n last (i.e., print last value)\n\nDefault value: sum \n\nIf there is only column, but multiple operations, all operations will be\napplied on that column. Likewise, if there is only one operation, but\nmultiple columns, that operation will be applied to all columns.\nOtherwise, the number of columns must match the the number of operations,\nand will be applied in respective order.\nE.g., \\"-c 5,4,6 -o sum,mean,count\\" will give the sum of column 5,\nthe mean of column 4, and the count of column 6.\nThe order of output columns will match the ordering given in the command.\n", + "required" : false, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + }, + { + "type" : "boolean_true", + "name" : "--full", + "description" : "Print all columns from input file. The first line in the group is used.\nDefault: print only grouped columns.\n", + "direction" : "input" + }, + { + "type" : "boolean_true", + "name" : "--inheader", + "description" : "Input file has a header line - the first line will be ignored.\n", + "direction" : "input" + }, + { + "type" : "boolean_true", + "name" : "--outheader", + "description" : "Print header line in the output, detailing the column names. \nIf the input file has headers (-inheader), the output file\nwill use the input's column names.\nIf the input file has no headers, the output file\nwill use \\"col_1\\", \\"col_2\\", etc. as the column names.\n", + "direction" : "input" + }, + { + "type" : "boolean_true", + "name" : "--header", + "description" : "same as '-inheader -outheader'.", + "direction" : "input" + }, + { + "type" : "boolean_true", + "name" : "--ignorecase", + "description" : "Group values regardless of upper/lower case.\n", + "direction" : "input" + }, + { + "type" : "integer", + "name" : "--precision", + "alternatives" : [ + "-prec" + ], + "description" : "Sets the decimal precision for output. \n", + "default" : [ + 5 + ], + "required" : false, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + }, + { + "type" : "string", + "name" : "--delimiter", + "alternatives" : [ + "-delim" + ], + "description" : "Specify a custom delimiter for the collapse operations.\n", + "example" : [ + "|" + ], + "default" : [ + "," + ], + "required" : false, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + } + ] + } + ], + "resources" : [ + { + "type" : "bash_script", + "path" : "script.sh", + "is_executable" : true + } + ], + "description" : "Summarizes a dataset column based upon common column groupings. \nAkin to the SQL \\"group by\\" command.\n", + "test_resources" : [ + { + "type" : "bash_script", + "path" : "test.sh", + "is_executable" : true + } + ], + "status" : "enabled", + "requirements" : { + "commands" : [ + "ps" + ] + }, + "keywords" : [ + "groupby", + "BED" + ], + "license" : "MIT", + "references" : { + "doi" : [ + "10.1093/bioinformatics/btq033" + ] + }, + "links" : { + "repository" : "https://github.com/arq5x/bedtools2", + "homepage" : "https://bedtools.readthedocs.io/en/latest/#", + "documentation" : "https://bedtools.readthedocs.io/en/latest/content/tools/groupby.html", + "issue_tracker" : "https://github.com/arq5x/bedtools2/issues" + }, + "runners" : [ + { + "type" : "executable", + "id" : "executable", + "docker_setup_strategy" : "ifneedbepullelsecachedbuild" + }, + { + "type" : "nextflow", + "id" : "nextflow", + "directives" : { + "tag" : "$id" + }, + "auto" : { + "simplifyInput" : true, + "simplifyOutput" : false, + "transcript" : false, + "publish" : false + }, + "config" : { + "labels" : { + "mem1gb" : "memory = 1000000000.B", + "mem2gb" : "memory = 2000000000.B", + "mem5gb" : "memory = 5000000000.B", + "mem10gb" : "memory = 10000000000.B", + "mem20gb" : "memory = 20000000000.B", + "mem50gb" : "memory = 50000000000.B", + "mem100gb" : "memory = 100000000000.B", + "mem200gb" : "memory = 200000000000.B", + "mem500gb" : "memory = 500000000000.B", + "mem1tb" : "memory = 1000000000000.B", + "mem2tb" : "memory = 2000000000000.B", + "mem5tb" : "memory = 5000000000000.B", + "mem10tb" : "memory = 10000000000000.B", + "mem20tb" : "memory = 20000000000000.B", + "mem50tb" : "memory = 50000000000000.B", + "mem100tb" : "memory = 100000000000000.B", + "mem200tb" : "memory = 200000000000000.B", + "mem500tb" : "memory = 500000000000000.B", + "mem1gib" : "memory = 1073741824.B", + "mem2gib" : "memory = 2147483648.B", + "mem4gib" : "memory = 4294967296.B", + "mem8gib" : "memory = 8589934592.B", + "mem16gib" : "memory = 17179869184.B", + "mem32gib" : "memory = 34359738368.B", + "mem64gib" : "memory = 68719476736.B", + "mem128gib" : "memory = 137438953472.B", + "mem256gib" : "memory = 274877906944.B", + "mem512gib" : "memory = 549755813888.B", + "mem1tib" : "memory = 1099511627776.B", + "mem2tib" : "memory = 2199023255552.B", + "mem4tib" : "memory = 4398046511104.B", + "mem8tib" : "memory = 8796093022208.B", + "mem16tib" : "memory = 17592186044416.B", + "mem32tib" : "memory = 35184372088832.B", + "mem64tib" : "memory = 70368744177664.B", + "mem128tib" : "memory = 140737488355328.B", + "mem256tib" : "memory = 281474976710656.B", + "mem512tib" : "memory = 562949953421312.B", + "cpu1" : "cpus = 1", + "cpu2" : "cpus = 2", + "cpu5" : "cpus = 5", + "cpu10" : "cpus = 10", + "cpu20" : "cpus = 20", + "cpu50" : "cpus = 50", + "cpu100" : "cpus = 100", + "cpu200" : "cpus = 200", + "cpu500" : "cpus = 500", + "cpu1000" : "cpus = 1000" + } + }, + "debug" : false, + "container" : "docker" + } + ], + "engines" : [ + { + "type" : "docker", + "id" : "docker", + "image" : "debian:stable-slim", + "target_registry" : "images.viash-hub.com", + "target_tag" : "main", + "namespace_separator" : "/", + "setup" : [ + { + "type" : "apt", + "packages" : [ + "bedtools", + "procps" + ], + "interactive" : false + }, + { + "type" : "docker", + "run" : [ + "echo \\"bedtools: \\\\\\"$(bedtools --version | sed -n 's/^bedtools //p')\\\\\\"\\" > /var/software_versions.txt\n" + ] + } + ] + }, + { + "type" : "native", + "id" : "native" + } + ], + "build_info" : { + "config" : "/workdir/root/repo/src/bedtools/bedtools_groupby/config.vsh.yaml", + "runner" : "nextflow", + "engine" : "docker|native", + "output" : "target/nextflow/bedtools/bedtools_groupby", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", + "git_remote" : "https://github.com/viash-hub/biobox" + }, + "package_config" : { + "name" : "biobox", + "version" : "main", + "description" : "A collection of bioinformatics tools for working with sequence data.\n", + "viash_version" : "0.9.0-RC7", + "source" : "src", + "target" : "target", + "config_mods" : [ + ".requirements.commands := ['ps']\n", + ".engines += { type: \\"native\\" }", + ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", + ".engines[.type == 'docker'].target_tag := 'main'" + ], + "keywords" : [ + "bioinformatics", + "modules", + "sequencing" + ], + "license" : "MIT", + "organization" : "vsh", + "links" : { + "repository" : "https://github.com/viash-hub/biobox", + "issue_tracker" : "https://github.com/viash-hub/biobox/issues" + } + } +}''')) +] + +// resolve dependencies dependencies (if any) + + +// inner workflow +// inner workflow hook +def innerWorkflowFactory(args) { + def rawScript = '''set -e +tempscript=".viash_script.sh" +cat > "$tempscript" << VIASHMAIN +#!/bin/bash + +## VIASH START +# The following code has been auto-generated by Viash. +$( if [ ! -z ${VIASH_PAR_INPUT+x} ]; then echo "${VIASH_PAR_INPUT}" | sed "s#'#'\\"'\\"'#g;s#.*#par_input='&'#" ; else echo "# par_input="; fi ) +$( if [ ! -z ${VIASH_PAR_OUTPUT+x} ]; then echo "${VIASH_PAR_OUTPUT}" | sed "s#'#'\\"'\\"'#g;s#.*#par_output='&'#" ; else echo "# par_output="; fi ) +$( if [ ! -z ${VIASH_PAR_GROUPBY+x} ]; then echo "${VIASH_PAR_GROUPBY}" | sed "s#'#'\\"'\\"'#g;s#.*#par_groupby='&'#" ; else echo "# par_groupby="; fi ) +$( if [ ! -z ${VIASH_PAR_COLUMN+x} ]; then echo "${VIASH_PAR_COLUMN}" | sed "s#'#'\\"'\\"'#g;s#.*#par_column='&'#" ; else echo "# par_column="; fi ) +$( if [ ! -z ${VIASH_PAR_OPERATION+x} ]; then echo "${VIASH_PAR_OPERATION}" | sed "s#'#'\\"'\\"'#g;s#.*#par_operation='&'#" ; else echo "# par_operation="; fi ) +$( if [ ! -z ${VIASH_PAR_FULL+x} ]; then echo "${VIASH_PAR_FULL}" | sed "s#'#'\\"'\\"'#g;s#.*#par_full='&'#" ; else echo "# par_full="; fi ) +$( if [ ! -z ${VIASH_PAR_INHEADER+x} ]; then echo "${VIASH_PAR_INHEADER}" | sed "s#'#'\\"'\\"'#g;s#.*#par_inheader='&'#" ; else echo "# par_inheader="; fi ) +$( if [ ! -z ${VIASH_PAR_OUTHEADER+x} ]; then echo "${VIASH_PAR_OUTHEADER}" | sed "s#'#'\\"'\\"'#g;s#.*#par_outheader='&'#" ; else echo "# par_outheader="; fi ) +$( if [ ! -z ${VIASH_PAR_HEADER+x} ]; then echo "${VIASH_PAR_HEADER}" | sed "s#'#'\\"'\\"'#g;s#.*#par_header='&'#" ; else echo "# par_header="; fi ) +$( if [ ! -z ${VIASH_PAR_IGNORECASE+x} ]; then echo "${VIASH_PAR_IGNORECASE}" | sed "s#'#'\\"'\\"'#g;s#.*#par_ignorecase='&'#" ; else echo "# par_ignorecase="; fi ) +$( if [ ! -z ${VIASH_PAR_PRECISION+x} ]; then echo "${VIASH_PAR_PRECISION}" | sed "s#'#'\\"'\\"'#g;s#.*#par_precision='&'#" ; else echo "# par_precision="; fi ) +$( if [ ! -z ${VIASH_PAR_DELIMITER+x} ]; then echo "${VIASH_PAR_DELIMITER}" | sed "s#'#'\\"'\\"'#g;s#.*#par_delimiter='&'#" ; else echo "# par_delimiter="; fi ) +$( if [ ! -z ${VIASH_META_NAME+x} ]; then echo "${VIASH_META_NAME}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_name='&'#" ; else echo "# meta_name="; fi ) +$( if [ ! -z ${VIASH_META_FUNCTIONALITY_NAME+x} ]; then echo "${VIASH_META_FUNCTIONALITY_NAME}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_functionality_name='&'#" ; else echo "# meta_functionality_name="; fi ) +$( if [ ! -z ${VIASH_META_RESOURCES_DIR+x} ]; then echo "${VIASH_META_RESOURCES_DIR}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_resources_dir='&'#" ; else echo "# meta_resources_dir="; fi ) +$( if [ ! -z ${VIASH_META_EXECUTABLE+x} ]; then echo "${VIASH_META_EXECUTABLE}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_executable='&'#" ; else echo "# meta_executable="; fi ) +$( if [ ! -z ${VIASH_META_CONFIG+x} ]; then echo "${VIASH_META_CONFIG}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_config='&'#" ; else echo "# meta_config="; fi ) +$( if [ ! -z ${VIASH_META_TEMP_DIR+x} ]; then echo "${VIASH_META_TEMP_DIR}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_temp_dir='&'#" ; else echo "# meta_temp_dir="; fi ) +$( if [ ! -z ${VIASH_META_CPUS+x} ]; then echo "${VIASH_META_CPUS}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_cpus='&'#" ; else echo "# meta_cpus="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_B+x} ]; then echo "${VIASH_META_MEMORY_B}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_b='&'#" ; else echo "# meta_memory_b="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_KB+x} ]; then echo "${VIASH_META_MEMORY_KB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_kb='&'#" ; else echo "# meta_memory_kb="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_MB+x} ]; then echo "${VIASH_META_MEMORY_MB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_mb='&'#" ; else echo "# meta_memory_mb="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_GB+x} ]; then echo "${VIASH_META_MEMORY_GB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_gb='&'#" ; else echo "# meta_memory_gb="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_TB+x} ]; then echo "${VIASH_META_MEMORY_TB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_tb='&'#" ; else echo "# meta_memory_tb="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_PB+x} ]; then echo "${VIASH_META_MEMORY_PB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_pb='&'#" ; else echo "# meta_memory_pb="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_KIB+x} ]; then echo "${VIASH_META_MEMORY_KIB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_kib='&'#" ; else echo "# meta_memory_kib="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_MIB+x} ]; then echo "${VIASH_META_MEMORY_MIB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_mib='&'#" ; else echo "# meta_memory_mib="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_GIB+x} ]; then echo "${VIASH_META_MEMORY_GIB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_gib='&'#" ; else echo "# meta_memory_gib="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_TIB+x} ]; then echo "${VIASH_META_MEMORY_TIB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_tib='&'#" ; else echo "# meta_memory_tib="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_PIB+x} ]; then echo "${VIASH_META_MEMORY_PIB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_pib='&'#" ; else echo "# meta_memory_pib="; fi ) + +## VIASH END + +# Exit on error +set -eo pipefail + +# Unset parameters +unset_if_false=( + par_full + par_inheader + par_outheader + par_header + par_ignorecase +) + +for par in \\${unset_if_false[@]}; do + test_val="\\${!par}" + [[ "\\$test_val" == "false" ]] && unset \\$par +done + +bedtools groupby \\\\ + \\${par_full:+-full} \\\\ + \\${par_inheader:+-inheader} \\\\ + \\${par_outheader:+-outheader} \\\\ + \\${par_header:+-header} \\\\ + \\${par_ignorecase:+-ignorecase} \\\\ + \\${par_precision:+-prec "\\$par_precision"} \\\\ + \\${par_delimiter:+-delim "\\$par_delimiter"} \\\\ + -i "\\$par_input" \\\\ + -g "\\$par_groupby" \\\\ + -c "\\$par_column" \\\\ + \\${par_operation:+-o "\\$par_operation"} \\\\ + > "\\$par_output" + +VIASHMAIN +bash "$tempscript" +''' + + return vdsl3WorkflowFactory(args, meta, rawScript) +} + + + +/** + * Generate a workflow for VDSL3 modules. + * + * This function is called by the workflowFactory() function. + * + * Input channel: [id, input_map] + * Output channel: [id, output_map] + * + * Internally, this workflow will convert the input channel + * to a format which the Nextflow module will be able to handle. + */ +def vdsl3WorkflowFactory(Map args, Map meta, String rawScript) { + def key = args["key"] + def processObj = null + + workflow processWf { + take: input_ + main: + + if (processObj == null) { + processObj = _vdsl3ProcessFactory(args, meta, rawScript) + } + + output_ = input_ + | map { tuple -> + def id = tuple[0] + def data_ = tuple[1] + + if (workflow.stubRun) { + // add id if missing + data_ = [id: 'stub'] + data_ + } + + // process input files separately + def inputPaths = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "input" } + .collect { par -> + def val = data_.containsKey(par.plainName) ? data_[par.plainName] : [] + def inputFiles = [] + if (val == null) { + inputFiles = [] + } else if (val instanceof List) { + inputFiles = val + } else if (val instanceof Path) { + inputFiles = [ val ] + } else { + inputFiles = [] + } + if (!workflow.stubRun) { + // throw error when an input file doesn't exist + inputFiles.each{ file -> + assert file.exists() : + "Error in module '${key}' id '${id}' argument '${par.plainName}'.\n" + + " Required input file does not exist.\n" + + " Path: '$file'.\n" + + " Expected input file to exist" + } + } + inputFiles + } + + // remove input files + def argsExclInputFiles = meta.config.allArguments + .findAll { (it.type != "file" || it.direction != "input") && data_.containsKey(it.plainName) } + .collectEntries { par -> + def parName = par.plainName + def val = data_[parName] + if (par.multiple && val instanceof Collection) { + val = val.join(par.multiple_sep) + } + if (par.direction == "output" && par.type == "file") { + val = val.replaceAll('\\$id', id).replaceAll('\\$key', key) + } + [parName, val] + } + + [ id ] + inputPaths + [ argsExclInputFiles, meta.resources_dir ] + } + | processObj + | map { output -> + def outputFiles = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "output" } + .indexed() + .collectEntries{ index, par -> + def out = output[index + 1] + // strip dummy '.exitcode' file from output (see nextflow-io/nextflow#2678) + if (!out instanceof List || out.size() <= 1) { + if (par.multiple) { + out = [] + } else { + assert !par.required : + "Error in module '${key}' id '${output[0]}' argument '${par.plainName}'.\n" + + " Required output file is missing" + out = null + } + } else if (out.size() == 2 && !par.multiple) { + out = out[1] + } else { + out = out.drop(1) + } + [ par.plainName, out ] + } + + // drop null outputs + outputFiles.removeAll{it.value == null} + + [ output[0], outputFiles ] + } + emit: output_ + } + + return processWf +} + +// depends on: session? +def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) { + // autodetect process key + def wfKey = workflowArgs["key"] + def procKeyPrefix = "${wfKey}_process" + def scriptMeta = nextflow.script.ScriptMeta.current() + def existing = scriptMeta.getProcessNames().findAll{it.startsWith(procKeyPrefix)} + def numbers = existing.collect{it.replace(procKeyPrefix, "0").toInteger()} + def newNumber = (numbers + [-1]).max() + 1 + + def procKey = newNumber == 0 ? procKeyPrefix : "$procKeyPrefix$newNumber" + + if (newNumber > 0) { + log.warn "Key for module '${wfKey}' is duplicated.\n", + "If you run a component multiple times in the same workflow,\n" + + "it's recommended you set a unique key for every call,\n" + + "for example: ${wfKey}.run(key: \"foo\")." + } + + // subset directives and convert to list of tuples + def drctv = workflowArgs.directives + + // TODO: unit test the two commands below + // convert publish array into tags + def valueToStr = { val -> + // ignore closures + if (val instanceof CharSequence) { + if (!val.matches('^[{].*[}]$')) { + '"' + val + '"' + } else { + val + } + } else if (val instanceof List) { + "[" + val.collect{valueToStr(it)}.join(", ") + "]" + } else if (val instanceof Map) { + "[" + val.collect{k, v -> k + ": " + valueToStr(v)}.join(", ") + "]" + } else { + val.inspect() + } + } + + // multiple entries allowed: label, publishdir + def drctvStrs = drctv.collect { key, value -> + if (key in ["label", "publishDir"]) { + value.collect{ val -> + if (val instanceof Map) { + "\n$key " + val.collect{ k, v -> k + ": " + valueToStr(v) }.join(", ") + } else if (val == null) { + "" + } else { + "\n$key " + valueToStr(val) + } + }.join() + } else if (value instanceof Map) { + "\n$key " + value.collect{ k, v -> k + ": " + valueToStr(v) }.join(", ") + } else { + "\n$key " + valueToStr(value) + } + }.join() + + def inputPaths = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "input" } + .collect { ', path(viash_par_' + it.plainName + ', stageAs: "_viash_par/' + it.plainName + '_?/*")' } + .join() + + def outputPaths = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "output" } + .collect { par -> + // insert dummy into every output (see nextflow-io/nextflow#2678) + if (!par.multiple) { + ', path{[".exitcode", args.' + par.plainName + ']}' + } else { + ', path{[".exitcode"] + args.' + par.plainName + '}' + } + } + .join() + + // TODO: move this functionality somewhere else? + if (workflowArgs.auto.transcript) { + outputPaths = outputPaths + ', path{[".exitcode", ".command*"]}' + } else { + outputPaths = outputPaths + ', path{[".exitcode"]}' + } + + // create dirs for output files (based on BashWrapper.createParentFiles) + def createParentStr = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "output" && it.create_parent } + .collect { par -> + "\${ args.containsKey(\"${par.plainName}\") ? \"mkdir_parent \\\"\" + (args[\"${par.plainName}\"] instanceof String ? args[\"${par.plainName}\"] : args[\"${par.plainName}\"].join('\" \"')) + \"\\\"\" : \"\" }" + } + .join("\n") + + // construct inputFileExports + def inputFileExports = meta.config.allArguments + .findAll { it.type == "file" && it.direction.toLowerCase() == "input" } + .collect { par -> + def viash_par_contents = "(viash_par_${par.plainName} instanceof List ? viash_par_${par.plainName}.join(\"${par.multiple_sep}\") : viash_par_${par.plainName})" + "\n\${viash_par_${par.plainName}.empty ? \"\" : \"export VIASH_PAR_${par.plainName.toUpperCase()}=\\\"\" + ${viash_par_contents} + \"\\\"\"}" + } + + // NOTE: if using docker, use /tmp instead of tmpDir! + def tmpDir = java.nio.file.Paths.get( + System.getenv('NXF_TEMP') ?: + System.getenv('VIASH_TEMP') ?: + System.getenv('VIASH_TMPDIR') ?: + System.getenv('VIASH_TEMPDIR') ?: + System.getenv('VIASH_TMP') ?: + System.getenv('TEMP') ?: + System.getenv('TMPDIR') ?: + System.getenv('TEMPDIR') ?: + System.getenv('TMP') ?: + '/tmp' + ).toAbsolutePath() + + // construct stub + def stub = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "output" } + .collect { par -> + "\${ args.containsKey(\"${par.plainName}\") ? \"touch2 \\\"\" + (args[\"${par.plainName}\"] instanceof String ? args[\"${par.plainName}\"].replace(\"_*\", \"_0\") : args[\"${par.plainName}\"].join('\" \"')) + \"\\\"\" : \"\" }" + } + .join("\n") + + // escape script + def escapedScript = rawScript.replace('\\', '\\\\').replace('$', '\\$').replace('"""', '\\"\\"\\"') + + // publishdir assert + def assertStr = (workflowArgs.auto.publish == true) || workflowArgs.auto.transcript ? + """\nassert task.publishDir.size() > 0: "if auto.publish is true, params.publish_dir needs to be defined.\\n Example: --publish_dir './output/'" """ : + "" + + // generate process string + def procStr = + """nextflow.enable.dsl=2 + | + |process $procKey {$drctvStrs + |input: + | tuple val(id)$inputPaths, val(args), path(resourcesDir, stageAs: ".viash_meta_resources") + |output: + | tuple val("\$id")$outputPaths, optional: true + |stub: + |\"\"\" + |touch2() { mkdir -p "\\\$(dirname "\\\$1")" && touch "\\\$1" ; } + |$stub + |\"\"\" + |script:$assertStr + |def escapeText = { s -> s.toString().replaceAll('([`"])', '\\\\\\\\\$1') } + |def parInject = args + | .findAll{key, value -> value != null} + | .collect{key, value -> "export VIASH_PAR_\${key.toUpperCase()}=\\\"\${escapeText(value)}\\\""} + | .join("\\n") + |\"\"\" + |# meta exports + |export VIASH_META_RESOURCES_DIR="\${resourcesDir}" + |export VIASH_META_TEMP_DIR="${['docker', 'podman', 'charliecloud'].any{ it == workflow.containerEngine } ? '/tmp' : tmpDir}" + |export VIASH_META_NAME="${meta.config.name}" + |# export VIASH_META_EXECUTABLE="\\\$VIASH_META_RESOURCES_DIR/\\\$VIASH_META_NAME" + |export VIASH_META_CONFIG="\\\$VIASH_META_RESOURCES_DIR/.config.vsh.yaml" + |\${task.cpus ? "export VIASH_META_CPUS=\$task.cpus" : "" } + |\${task.memory?.bytes != null ? "export VIASH_META_MEMORY_B=\$task.memory.bytes" : "" } + |if [ ! -z \\\${VIASH_META_MEMORY_B+x} ]; then + | export VIASH_META_MEMORY_KB=\\\$(( (\\\$VIASH_META_MEMORY_B+999) / 1000 )) + | export VIASH_META_MEMORY_MB=\\\$(( (\\\$VIASH_META_MEMORY_KB+999) / 1000 )) + | export VIASH_META_MEMORY_GB=\\\$(( (\\\$VIASH_META_MEMORY_MB+999) / 1000 )) + | export VIASH_META_MEMORY_TB=\\\$(( (\\\$VIASH_META_MEMORY_GB+999) / 1000 )) + | export VIASH_META_MEMORY_PB=\\\$(( (\\\$VIASH_META_MEMORY_TB+999) / 1000 )) + | export VIASH_META_MEMORY_KIB=\\\$(( (\\\$VIASH_META_MEMORY_B+1023) / 1024 )) + | export VIASH_META_MEMORY_MIB=\\\$(( (\\\$VIASH_META_MEMORY_KIB+1023) / 1024 )) + | export VIASH_META_MEMORY_GIB=\\\$(( (\\\$VIASH_META_MEMORY_MIB+1023) / 1024 )) + | export VIASH_META_MEMORY_TIB=\\\$(( (\\\$VIASH_META_MEMORY_GIB+1023) / 1024 )) + | export VIASH_META_MEMORY_PIB=\\\$(( (\\\$VIASH_META_MEMORY_TIB+1023) / 1024 )) + |fi + | + |# meta synonyms + |export VIASH_TEMP="\\\$VIASH_META_TEMP_DIR" + |export TEMP_DIR="\\\$VIASH_META_TEMP_DIR" + | + |# create output dirs if need be + |function mkdir_parent { + | for file in "\\\$@"; do + | mkdir -p "\\\$(dirname "\\\$file")" + | done + |} + |$createParentStr + | + |# argument exports${inputFileExports.join()} + |\$parInject + | + |# process script + |${escapedScript} + |\"\"\" + |} + |""".stripMargin() + + // TODO: print on debug + // if (workflowArgs.debug == true) { + // println("######################\n$procStr\n######################") + // } + + // write process to temp file + def tempFile = java.nio.file.Files.createTempFile("viash-process-${procKey}-", ".nf") + addShutdownHook { java.nio.file.Files.deleteIfExists(tempFile) } + tempFile.text = procStr + + // create process from temp file + def binding = new nextflow.script.ScriptBinding([:]) + def session = nextflow.Nextflow.getSession() + def parser = new nextflow.script.ScriptParser(session) + .setModule(true) + .setBinding(binding) + def moduleScript = parser.runScript(tempFile) + .getScript() + + // register module in meta + def module = new nextflow.script.IncludeDef.Module(name: procKey) + scriptMeta.addModule(moduleScript, module.name, module.alias) + + // retrieve and return process from meta + return scriptMeta.getProcess(procKey) +} + +// defaults +meta["defaults"] = [ + // key to be used to trace the process and determine output names + key: null, + + // fixed arguments to be passed to script + args: [:], + + // default directives + directives: readJsonBlob('''{ + "container" : { + "registry" : "images.viash-hub.com", + "image" : "vsh/biobox/bedtools/bedtools_groupby", + "tag" : "main" + }, + "tag" : "$id" +}'''), + + // auto settings + auto: readJsonBlob('''{ + "simplifyInput" : true, + "simplifyOutput" : false, + "transcript" : false, + "publish" : false +}'''), + + // Apply a map over the incoming tuple + // Example: `{ tup -> [ tup[0], [input: tup[1].output] ] + tup.drop(2) }` + map: null, + + // Apply a map over the ID element of a tuple (i.e. the first element) + // Example: `{ id -> id + "_foo" }` + mapId: null, + + // Apply a map over the data element of a tuple (i.e. the second element) + // Example: `{ data -> [ input: data.output ] }` + mapData: null, + + // Apply a map over the passthrough elements of a tuple (i.e. the tuple excl. the first two elements) + // Example: `{ pt -> pt.drop(1) }` + mapPassthrough: null, + + // Filter the channel + // Example: `{ tup -> tup[0] == "foo" }` + filter: null, + + // Choose whether or not to run the component on the tuple if the condition is true. + // Otherwise, the tuple will be passed through. + // Example: `{ tup -> tup[0] != "skip_this" }` + runIf: null, + + // Rename keys in the data field of the tuple (i.e. the second element) + // Will likely be deprecated in favour of `fromState`. + // Example: `[ "new_key": "old_key" ]` + renameKeys: null, + + // Fetch data from the state and pass it to the module without altering the current state. + // + // `fromState` should be `null`, `List[String]`, `Map[String, String]` or a function. + // + // - If it is `null`, the state will be passed to the module as is. + // - If it is a `List[String]`, the data will be the values of the state at the given keys. + // - If it is a `Map[String, String]`, the data will be the values of the state at the given keys, with the keys renamed according to the map. + // - If it is a function, the tuple (`[id, state]`) in the channel will be passed to the function, and the result will be used as the data. + // + // Example: `{ id, state -> [input: state.fastq_file] }` + // Default: `null` + fromState: null, + + // Determine how the state should be updated after the module has been run. + // + // `toState` should be `null`, `List[String]`, `Map[String, String]` or a function. + // + // - If it is `null`, the state will be replaced with the output of the module. + // - If it is a `List[String]`, the state will be updated with the values of the data at the given keys. + // - If it is a `Map[String, String]`, the state will be updated with the values of the data at the given keys, with the keys renamed according to the map. + // - If it is a function, a tuple (`[id, output, state]`) will be passed to the function, and the result will be used as the new state. + // + // Example: `{ id, output, state -> state + [counts: state.output] }` + // Default: `{ id, output, state -> output }` + toState: null, + + // Whether or not to print debug messages + // Default: `false` + debug: false +] + +// initialise default workflow +meta["workflow"] = workflowFactory([key: meta.config.name], meta.defaults, meta) + +// add workflow to environment +nextflow.script.ScriptMeta.current().addDefinition(meta.workflow) + +// anonymous workflow for running this module as a standalone +workflow { + // add id argument if it's not already in the config + // TODO: deep copy + def newConfig = deepClone(meta.config) + def newParams = deepClone(params) + + def argsContainsId = newConfig.allArguments.any{it.plainName == "id"} + if (!argsContainsId) { + def idArg = [ + 'name': '--id', + 'required': false, + 'type': 'string', + 'description': 'A unique id for every entry.', + 'multiple': false + ] + newConfig.arguments.add(0, idArg) + newConfig = processConfig(newConfig) + } + if (!newParams.containsKey("id")) { + newParams.id = "run" + } + + helpMessage(newConfig) + + channelFromParams(newParams, newConfig) + // make sure id is not in the state if id is not in the args + | map {id, state -> + if (!argsContainsId) { + [id, state.findAll{k, v -> k != "id"}] + } else { + [id, state] + } + } + | meta.workflow.run( + auto: [ publish: "state" ] + ) +} + +// END COMPONENT-SPECIFIC CODE diff --git a/target/nextflow/bedtools/bedtools_groupby/nextflow.config b/target/nextflow/bedtools/bedtools_groupby/nextflow.config new file mode 100644 index 00000000..d4a26dd2 --- /dev/null +++ b/target/nextflow/bedtools/bedtools_groupby/nextflow.config @@ -0,0 +1,126 @@ +manifest { + name = 'bedtools/bedtools_groupby' + mainScript = 'main.nf' + nextflowVersion = '!>=20.12.1-edge' + version = 'main' + description = 'Summarizes a dataset column based upon common column groupings. \nAkin to the SQL "group by" command.\n' + author = 'Theodoro Gasperin Terra Camargo' +} + +process.container = 'nextflow/bash:latest' + +// detect tempdir +tempDir = java.nio.file.Paths.get( + System.getenv('NXF_TEMP') ?: + System.getenv('VIASH_TEMP') ?: + System.getenv('TEMPDIR') ?: + System.getenv('TMPDIR') ?: + '/tmp' +).toAbsolutePath() + +profiles { + no_publish { + process { + withName: '.*' { + publishDir = [ + enabled: false + ] + } + } + } + mount_temp { + docker.temp = tempDir + podman.temp = tempDir + charliecloud.temp = tempDir + } + docker { + docker.enabled = true + // docker.userEmulation = true + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + singularity { + singularity.enabled = true + singularity.autoMounts = true + docker.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + podman { + podman.enabled = true + docker.enabled = false + singularity.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + shifter { + shifter.enabled = true + docker.enabled = false + singularity.enabled = false + podman.enabled = false + charliecloud.enabled = false + } + charliecloud { + charliecloud.enabled = true + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + } +} + +process{ + withLabel: mem1gb { memory = 1000000000.B } + withLabel: mem2gb { memory = 2000000000.B } + withLabel: mem5gb { memory = 5000000000.B } + withLabel: mem10gb { memory = 10000000000.B } + withLabel: mem20gb { memory = 20000000000.B } + withLabel: mem50gb { memory = 50000000000.B } + withLabel: mem100gb { memory = 100000000000.B } + withLabel: mem200gb { memory = 200000000000.B } + withLabel: mem500gb { memory = 500000000000.B } + withLabel: mem1tb { memory = 1000000000000.B } + withLabel: mem2tb { memory = 2000000000000.B } + withLabel: mem5tb { memory = 5000000000000.B } + withLabel: mem10tb { memory = 10000000000000.B } + withLabel: mem20tb { memory = 20000000000000.B } + withLabel: mem50tb { memory = 50000000000000.B } + withLabel: mem100tb { memory = 100000000000000.B } + withLabel: mem200tb { memory = 200000000000000.B } + withLabel: mem500tb { memory = 500000000000000.B } + withLabel: mem1gib { memory = 1073741824.B } + withLabel: mem2gib { memory = 2147483648.B } + withLabel: mem4gib { memory = 4294967296.B } + withLabel: mem8gib { memory = 8589934592.B } + withLabel: mem16gib { memory = 17179869184.B } + withLabel: mem32gib { memory = 34359738368.B } + withLabel: mem64gib { memory = 68719476736.B } + withLabel: mem128gib { memory = 137438953472.B } + withLabel: mem256gib { memory = 274877906944.B } + withLabel: mem512gib { memory = 549755813888.B } + withLabel: mem1tib { memory = 1099511627776.B } + withLabel: mem2tib { memory = 2199023255552.B } + withLabel: mem4tib { memory = 4398046511104.B } + withLabel: mem8tib { memory = 8796093022208.B } + withLabel: mem16tib { memory = 17592186044416.B } + withLabel: mem32tib { memory = 35184372088832.B } + withLabel: mem64tib { memory = 70368744177664.B } + withLabel: mem128tib { memory = 140737488355328.B } + withLabel: mem256tib { memory = 281474976710656.B } + withLabel: mem512tib { memory = 562949953421312.B } + withLabel: cpu1 { cpus = 1 } + withLabel: cpu2 { cpus = 2 } + withLabel: cpu5 { cpus = 5 } + withLabel: cpu10 { cpus = 10 } + withLabel: cpu20 { cpus = 20 } + withLabel: cpu50 { cpus = 50 } + withLabel: cpu100 { cpus = 100 } + withLabel: cpu200 { cpus = 200 } + withLabel: cpu500 { cpus = 500 } + withLabel: cpu1000 { cpus = 1000 } +} + + diff --git a/target/nextflow/bedtools/bedtools_groupby/nextflow_schema.json b/target/nextflow/bedtools/bedtools_groupby/nextflow_schema.json new file mode 100644 index 00000000..a4d92e23 --- /dev/null +++ b/target/nextflow/bedtools/bedtools_groupby/nextflow_schema.json @@ -0,0 +1,216 @@ +{ +"$schema": "http://json-schema.org/draft-07/schema", +"title": "bedtools_groupby", +"description": "Summarizes a dataset column based upon common column groupings. \nAkin to the SQL \"group by\" command.\n", +"type": "object", +"definitions": { + + + + "inputs" : { + "title": "Inputs", + "type": "object", + "description": "No description", + "properties": { + + + "input": { + "type": + "string", + "description": "Type: `file`, required, example: `input_a.bed`. The input BED file to be used", + "help_text": "Type: `file`, required, example: `input_a.bed`. The input BED file to be used.\n" + + } + + +} +}, + + + "outputs" : { + "title": "Outputs", + "type": "object", + "description": "No description", + "properties": { + + + "output": { + "type": + "string", + "description": "Type: `file`, required, default: `$id.$key.output.bed`, example: `output.bed`. The output groupby BED file", + "help_text": "Type: `file`, required, default: `$id.$key.output.bed`, example: `output.bed`. The output groupby BED file. \n" + , + "default": "$id.$key.output.bed" + } + + +} +}, + + + "options" : { + "title": "Options", + "type": "object", + "description": "No description", + "properties": { + + + "groupby": { + "type": + "string", + "description": "Type: `string`, required. Specify the columns (1-based) for the grouping", + "help_text": "Type: `string`, required. Specify the columns (1-based) for the grouping.\nThe columns must be comma separated.\n- Default: 1,2,3 \n" + + } + + + , + "column": { + "type": + "integer", + "description": "Type: `integer`, required. Specify the column (1-based) that should be summarized", + "help_text": "Type: `integer`, required. Specify the column (1-based) that should be summarized.\n" + + } + + + , + "operation": { + "type": + "string", + "description": "Type: `string`. Specify the operation that should be applied to opCol", + "help_text": "Type: `string`. Specify the operation that should be applied to opCol.\nValid operations:\n sum, count, count_distinct, min, max,\n mean, median, mode, antimode,\n stdev, sstdev (sample standard dev.),\n collapse (i.e., print a comma separated list (duplicates allowed)), \n distinct (i.e., print a comma separated list (NO duplicates allowed)), \n distinct_sort_num (as distinct, but sorted numerically, ascending), \n distinct_sort_num_desc (as distinct, but sorted numerically, descending), \n concat (i.e., merge values into a single, non-delimited string), \n freqdesc (i.e., print desc. list of values:freq)\n freqasc (i.e., print asc. list of values:freq)\n first (i.e., print first value)\n last (i.e., print last value)\n\nDefault value: sum \n\nIf there is only column, but multiple operations, all operations will be\napplied on that column. Likewise, if there is only one operation, but\nmultiple columns, that operation will be applied to all columns.\nOtherwise, the number of columns must match the the number of operations,\nand will be applied in respective order.\nE.g., \"-c 5,4,6 -o sum,mean,count\" will give the sum of column 5,\nthe mean of column 4, and the count of column 6.\nThe order of output columns will match the ordering given in the command.\n" + + } + + + , + "full": { + "type": + "boolean", + "description": "Type: `boolean_true`, default: `false`. Print all columns from input file", + "help_text": "Type: `boolean_true`, default: `false`. Print all columns from input file. The first line in the group is used.\nDefault: print only grouped columns.\n" + , + "default": "False" + } + + + , + "inheader": { + "type": + "boolean", + "description": "Type: `boolean_true`, default: `false`. Input file has a header line - the first line will be ignored", + "help_text": "Type: `boolean_true`, default: `false`. Input file has a header line - the first line will be ignored.\n" + , + "default": "False" + } + + + , + "outheader": { + "type": + "boolean", + "description": "Type: `boolean_true`, default: `false`. Print header line in the output, detailing the column names", + "help_text": "Type: `boolean_true`, default: `false`. Print header line in the output, detailing the column names. \nIf the input file has headers (-inheader), the output file\nwill use the input\u0027s column names.\nIf the input file has no headers, the output file\nwill use \"col_1\", \"col_2\", etc. as the column names.\n" + , + "default": "False" + } + + + , + "header": { + "type": + "boolean", + "description": "Type: `boolean_true`, default: `false`. same as \u0027-inheader -outheader\u0027", + "help_text": "Type: `boolean_true`, default: `false`. same as \u0027-inheader -outheader\u0027." + , + "default": "False" + } + + + , + "ignorecase": { + "type": + "boolean", + "description": "Type: `boolean_true`, default: `false`. Group values regardless of upper/lower case", + "help_text": "Type: `boolean_true`, default: `false`. Group values regardless of upper/lower case.\n" + , + "default": "False" + } + + + , + "precision": { + "type": + "integer", + "description": "Type: `integer`, default: `5`. Sets the decimal precision for output", + "help_text": "Type: `integer`, default: `5`. Sets the decimal precision for output. \n" + , + "default": "5" + } + + + , + "delimiter": { + "type": + "string", + "description": "Type: `string`, default: `,`, example: `|`. Specify a custom delimiter for the collapse operations", + "help_text": "Type: `string`, default: `,`, example: `|`. Specify a custom delimiter for the collapse operations.\n" + , + "default": "," + } + + +} +}, + + + "nextflow input-output arguments" : { + "title": "Nextflow input-output arguments", + "type": "object", + "description": "Input/output parameters for Nextflow itself. Please note that both publishDir and publish_dir are supported but at least one has to be configured.", + "properties": { + + + "publish_dir": { + "type": + "string", + "description": "Type: `string`, required, example: `output/`. Path to an output directory", + "help_text": "Type: `string`, required, example: `output/`. Path to an output directory." + + } + + + , + "param_list": { + "type": + "string", + "description": "Type: `string`, example: `my_params.yaml`. Allows inputting multiple parameter sets to initialise a Nextflow channel", + "help_text": "Type: `string`, example: `my_params.yaml`. Allows inputting multiple parameter sets to initialise a Nextflow channel. A `param_list` can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob.\n\n* A list of maps (as-is) where the keys of each map corresponds to the arguments of the pipeline. Example: in a `nextflow.config` file: `param_list: [ [\u0027id\u0027: \u0027foo\u0027, \u0027input\u0027: \u0027foo.txt\u0027], [\u0027id\u0027: \u0027bar\u0027, \u0027input\u0027: \u0027bar.txt\u0027] ]`.\n* A csv file should have column names which correspond to the different arguments of this pipeline. Example: `--param_list data.csv` with columns `id,input`.\n* A json or a yaml file should be a list of maps, each of which has keys corresponding to the arguments of the pipeline. Example: `--param_list data.json` with contents `[ {\u0027id\u0027: \u0027foo\u0027, \u0027input\u0027: \u0027foo.txt\u0027}, {\u0027id\u0027: \u0027bar\u0027, \u0027input\u0027: \u0027bar.txt\u0027} ]`.\n* A yaml blob can also be passed directly as a string. Example: `--param_list \"[ {\u0027id\u0027: \u0027foo\u0027, \u0027input\u0027: \u0027foo.txt\u0027}, {\u0027id\u0027: \u0027bar\u0027, \u0027input\u0027: \u0027bar.txt\u0027} ]\"`.\n\nWhen passing a csv, json or yaml file, relative path names are relativized to the location of the parameter file. No relativation is performed when `param_list` is a list of maps (as-is) or a yaml blob.", + "hidden": true + + } + + +} +} +}, +"allOf": [ + + { + "$ref": "#/definitions/inputs" + }, + + { + "$ref": "#/definitions/outputs" + }, + + { + "$ref": "#/definitions/options" + }, + + { + "$ref": "#/definitions/nextflow input-output arguments" + } +] +} diff --git a/target/nextflow/bedtools/bedtools_intersect/.config.vsh.yaml b/target/nextflow/bedtools/bedtools_intersect/.config.vsh.yaml index a6340d80..288ecf40 100644 --- a/target/nextflow/bedtools/bedtools_intersect/.config.vsh.yaml +++ b/target/nextflow/bedtools/bedtools_intersect/.config.vsh.yaml @@ -409,15 +409,15 @@ build_info: engine: "docker|native" output: "target/nextflow/bedtools/bedtools_intersect" executable: "target/nextflow/bedtools/bedtools_intersect/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/bedtools/bedtools_intersect/main.nf b/target/nextflow/bedtools/bedtools_intersect/main.nf index 10af2709..a4251af2 100644 --- a/target/nextflow/bedtools/bedtools_intersect/main.nf +++ b/target/nextflow/bedtools/bedtools_intersect/main.nf @@ -1,6 +1,6 @@ // bedtools_intersect main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -763,8 +763,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -786,7 +789,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -824,8 +840,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1601,8 +1620,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2629,30 +2648,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2723,7 +2743,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -3255,15 +3275,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/bedtools/bedtools_intersect", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/bedtools/bedtools_intersect/nextflow_schema.json b/target/nextflow/bedtools/bedtools_intersect/nextflow_schema.json index 9ef264fa..add67b19 100644 --- a/target/nextflow/bedtools/bedtools_intersect/nextflow_schema.json +++ b/target/nextflow/bedtools/bedtools_intersect/nextflow_schema.json @@ -27,8 +27,8 @@ "input_b": { "type": "string", - "description": "Type: List of `file`, required, example: `input_b.bed`, multiple_sep: `\":\"`. The input file(s) (BED/GFF/VCF/BAM) to be used as the -b file(s)", - "help_text": "Type: List of `file`, required, example: `input_b.bed`, multiple_sep: `\":\"`. The input file(s) (BED/GFF/VCF/BAM) to be used as the -b file(s).\n" + "description": "Type: List of `file`, required, example: `input_b.bed`, multiple_sep: `\";\"`. The input file(s) (BED/GFF/VCF/BAM) to be used as the -b file(s)", + "help_text": "Type: List of `file`, required, example: `input_b.bed`, multiple_sep: `\";\"`. The input file(s) (BED/GFF/VCF/BAM) to be used as the -b file(s).\n" } diff --git a/target/nextflow/bedtools/bedtools_links/.config.vsh.yaml b/target/nextflow/bedtools/bedtools_links/.config.vsh.yaml new file mode 100644 index 00000000..2789d8e1 --- /dev/null +++ b/target/nextflow/bedtools/bedtools_links/.config.vsh.yaml @@ -0,0 +1,236 @@ +name: "bedtools_links" +namespace: "bedtools" +version: "main" +authors: +- name: "Theodoro Gasperin Terra Camargo" + roles: + - "author" + - "maintainer" + info: + links: + email: "theodorogtc@gmail.com" + github: "tgaspe" + linkedin: "theodoro-gasperin-terra-camargo" + organizations: + - name: "Data Intuitive" + href: "https://www.data-intuitive.com" + role: "Bioinformatician" +argument_groups: +- name: "Inputs" + arguments: + - type: "file" + name: "--input" + alternatives: + - "-i" + description: "Input file (bed/gff/vcf)." + info: null + must_exist: true + create_parent: true + required: true + direction: "input" + multiple: false + multiple_sep: ";" +- name: "Outputs" + arguments: + - type: "file" + name: "--output" + alternatives: + - "-o" + description: "Output HTML file to be written." + info: null + must_exist: true + create_parent: true + required: false + direction: "output" + multiple: false + multiple_sep: ";" +- name: "Options" + description: "By default, the links created will point to human (hg18) UCSC browser.\n\ + If you have a local mirror, you can override this behavior by supplying\nthe -base,\ + \ -org, and -db options.\n\nFor example, if the URL of your local mirror for mouse\ + \ MM9 is called: \nhttp://mymirror.myuniversity.edu, then you would use the following:\n\ + --base_url http://mymirror.myuniversity.edu\n--organism mouse\n--database mm9\n" + arguments: + - type: "string" + name: "--base_url" + alternatives: + - "-base" + description: "The “basename” for the UCSC browser.\n" + info: null + default: + - "http://genome.ucsc.edu" + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "string" + name: "--organism" + alternatives: + - "-org" + description: "The organism (e.g. mouse, human). \n" + info: null + default: + - "human" + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "string" + name: "--database" + alternatives: + - "-db" + description: "The genome build. \n" + info: null + default: + - "hg18" + required: false + direction: "input" + multiple: false + multiple_sep: ";" +resources: +- type: "bash_script" + path: "script.sh" + is_executable: true +description: "Creates an HTML file with links to an instance of the UCSC Genome Browser\ + \ for all features / intervals in a file. \nThis is useful for cases when one wants\ + \ to manually inspect through a large set of annotations or features.\n" +test_resources: +- type: "bash_script" + path: "test.sh" + is_executable: true +info: null +status: "enabled" +requirements: + commands: + - "ps" +keywords: +- "Links" +- "BED" +- "GFF" +- "VCF" +license: "MIT" +references: + doi: + - "10.1093/bioinformatics/btq033" +links: + repository: "https://github.com/arq5x/bedtools2" + homepage: "https://bedtools.readthedocs.io/en/latest/#" + documentation: "https://bedtools.readthedocs.io/en/latest/content/tools/links.html" + issue_tracker: "https://github.com/arq5x/bedtools2/issues" +runners: +- type: "executable" + id: "executable" + docker_setup_strategy: "ifneedbepullelsecachedbuild" +- type: "nextflow" + id: "nextflow" + directives: + tag: "$id" + auto: + simplifyInput: true + simplifyOutput: false + transcript: false + publish: false + config: + labels: + mem1gb: "memory = 1000000000.B" + mem2gb: "memory = 2000000000.B" + mem5gb: "memory = 5000000000.B" + mem10gb: "memory = 10000000000.B" + mem20gb: "memory = 20000000000.B" + mem50gb: "memory = 50000000000.B" + mem100gb: "memory = 100000000000.B" + mem200gb: "memory = 200000000000.B" + mem500gb: "memory = 500000000000.B" + mem1tb: "memory = 1000000000000.B" + mem2tb: "memory = 2000000000000.B" + mem5tb: "memory = 5000000000000.B" + mem10tb: "memory = 10000000000000.B" + mem20tb: "memory = 20000000000000.B" + mem50tb: "memory = 50000000000000.B" + mem100tb: "memory = 100000000000000.B" + mem200tb: "memory = 200000000000000.B" + mem500tb: "memory = 500000000000000.B" + mem1gib: "memory = 1073741824.B" + mem2gib: "memory = 2147483648.B" + mem4gib: "memory = 4294967296.B" + mem8gib: "memory = 8589934592.B" + mem16gib: "memory = 17179869184.B" + mem32gib: "memory = 34359738368.B" + mem64gib: "memory = 68719476736.B" + mem128gib: "memory = 137438953472.B" + mem256gib: "memory = 274877906944.B" + mem512gib: "memory = 549755813888.B" + mem1tib: "memory = 1099511627776.B" + mem2tib: "memory = 2199023255552.B" + mem4tib: "memory = 4398046511104.B" + mem8tib: "memory = 8796093022208.B" + mem16tib: "memory = 17592186044416.B" + mem32tib: "memory = 35184372088832.B" + mem64tib: "memory = 70368744177664.B" + mem128tib: "memory = 140737488355328.B" + mem256tib: "memory = 281474976710656.B" + mem512tib: "memory = 562949953421312.B" + cpu1: "cpus = 1" + cpu2: "cpus = 2" + cpu5: "cpus = 5" + cpu10: "cpus = 10" + cpu20: "cpus = 20" + cpu50: "cpus = 50" + cpu100: "cpus = 100" + cpu200: "cpus = 200" + cpu500: "cpus = 500" + cpu1000: "cpus = 1000" + debug: false + container: "docker" +engines: +- type: "docker" + id: "docker" + image: "debian:stable-slim" + target_registry: "images.viash-hub.com" + target_tag: "main" + namespace_separator: "/" + setup: + - type: "apt" + packages: + - "bedtools" + - "procps" + interactive: false + - type: "docker" + run: + - "echo \"bedtools: \\\"$(bedtools --version | sed -n 's/^bedtools //p')\\\"\"\ + \ > /var/software_versions.txt\n" + entrypoint: [] + cmd: null +- type: "native" + id: "native" +build_info: + config: "src/bedtools/bedtools_links/config.vsh.yaml" + runner: "nextflow" + engine: "docker|native" + output: "target/nextflow/bedtools/bedtools_links" + executable: "target/nextflow/bedtools/bedtools_links/main.nf" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" + git_remote: "https://github.com/viash-hub/biobox" +package_config: + name: "biobox" + version: "main" + description: "A collection of bioinformatics tools for working with sequence data.\n" + info: null + viash_version: "0.9.0-RC7" + source: "src" + target: "target" + config_mods: + - ".requirements.commands := ['ps']\n" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'main'" + keywords: + - "bioinformatics" + - "modules" + - "sequencing" + license: "MIT" + organization: "vsh" + links: + repository: "https://github.com/viash-hub/biobox" + issue_tracker: "https://github.com/viash-hub/biobox/issues" diff --git a/target/nextflow/bedtools/bedtools_links/main.nf b/target/nextflow/bedtools/bedtools_links/main.nf new file mode 100644 index 00000000..7aed5f9b --- /dev/null +++ b/target/nextflow/bedtools/bedtools_links/main.nf @@ -0,0 +1,3624 @@ +// bedtools_links main +// +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a +// derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from +// Data Intuitive. +// +// The component may contain files which fall under a different license. The +// authors of this component should specify the license in the header of such +// files, or include a separate license file detailing the licenses of all included +// files. +// +// Component authors: +// * Theodoro Gasperin Terra Camargo (author, maintainer) + +//////////////////////////// +// VDSL3 helper functions // +//////////////////////////// + +// helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_checkArgumentType.nf' +class UnexpectedArgumentTypeException extends Exception { + String errorIdentifier + String stage + String plainName + String expectedClass + String foundClass + + // ${key ? " in module '$key'" : ""}${id ? " id '$id'" : ""} + UnexpectedArgumentTypeException(String errorIdentifier, String stage, String plainName, String expectedClass, String foundClass) { + super("Error${errorIdentifier ? " $errorIdentifier" : ""}:${stage ? " $stage" : "" } argument '${plainName}' has the wrong type. " + + "Expected type: ${expectedClass}. Found type: ${foundClass}") + this.errorIdentifier = errorIdentifier + this.stage = stage + this.plainName = plainName + this.expectedClass = expectedClass + this.foundClass = foundClass + } +} + +/** + * Checks if the given value is of the expected type. If not, an exception is thrown. + * + * @param stage The stage of the argument (input or output) + * @param par The parameter definition + * @param value The value to check + * @param errorIdentifier The identifier to use in the error message + * @return The value, if it is of the expected type + * @throws UnexpectedArgumentTypeException If the value is not of the expected type +*/ +def _checkArgumentType(String stage, Map par, Object value, String errorIdentifier) { + // expectedClass will only be != null if value is not of the expected type + def expectedClass = null + def foundClass = null + + // todo: split if need be + + if (!par.required && value == null) { + expectedClass = null + } else if (par.multiple) { + if (value !instanceof Collection) { + value = [value] + } + + // split strings + value = value.collectMany{ val -> + if (val instanceof String) { + // collect() to ensure that the result is a List and not simply an array + val.split(par.multiple_sep).collect() + } else { + [val] + } + } + + // process globs + if (par.type == "file" && par.direction == "input") { + value = value.collect{ it instanceof String ? file(it, hidden: true) : it }.flatten() + } + + // check types of elements in list + try { + value = value.collect { listVal -> + _checkArgumentType(stage, par + [multiple: false], listVal, errorIdentifier) + } + } catch (UnexpectedArgumentTypeException e) { + expectedClass = "List[${e.expectedClass}]" + foundClass = "List[${e.foundClass}]" + } + } else if (par.type == "string") { + // cast to string if need be + if (value instanceof GString) { + value = value.toString() + } + expectedClass = value instanceof String ? null : "String" + } else if (par.type == "integer") { + // cast to integer if need be + if (value instanceof String) { + try { + value = value.toInteger() + } catch (NumberFormatException e) { + // do nothing + } + } + if (value instanceof java.math.BigInteger) { + value = value.intValue() + } + expectedClass = value instanceof Integer ? null : "Integer" + } else if (par.type == "long") { + // cast to long if need be + if (value instanceof String) { + try { + value = value.toLong() + } catch (NumberFormatException e) { + // do nothing + } + } + if (value instanceof Integer) { + value = value.toLong() + } + expectedClass = value instanceof Long ? null : "Long" + } else if (par.type == "double") { + // cast to double if need be + if (value instanceof String) { + try { + value = value.toDouble() + } catch (NumberFormatException e) { + // do nothing + } + } + if (value instanceof java.math.BigDecimal) { + value = value.doubleValue() + } + if (value instanceof Float) { + value = value.toDouble() + } + expectedClass = value instanceof Double ? null : "Double" + } else if (par.type == "boolean" | par.type == "boolean_true" | par.type == "boolean_false") { + // cast to boolean if need be + if (value instanceof String) { + def valueLower = value.toLowerCase() + if (valueLower == "true") { + value = true + } else if (valueLower == "false") { + value = false + } + } + expectedClass = value instanceof Boolean ? null : "Boolean" + } else if (par.type == "file" && (par.direction == "input" || stage == "output")) { + // cast to path if need be + if (value instanceof String) { + value = file(value, hidden: true) + } + if (value instanceof File) { + value = value.toPath() + } + expectedClass = value instanceof Path ? null : "Path" + } else if (par.type == "file" && stage == "input" && par.direction == "output") { + // cast to string if need be + if (value instanceof GString) { + value = value.toString() + } + expectedClass = value instanceof String ? null : "String" + } else { + // didn't find a match for par.type + expectedClass = par.type + } + + if (expectedClass != null) { + if (foundClass == null) { + foundClass = value.getClass().getName() + } + throw new UnexpectedArgumentTypeException(errorIdentifier, stage, par.plainName, expectedClass, foundClass) + } + + return value +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_processInputValues.nf' +Map _processInputValues(Map inputs, Map config, String id, String key) { + if (!workflow.stubRun) { + config.allArguments.each { arg -> + if (arg.required) { + assert inputs.containsKey(arg.plainName) && inputs.get(arg.plainName) != null : + "Error in module '${key}' id '${id}': required input argument '${arg.plainName}' is missing" + } + } + + inputs = inputs.collectEntries { name, value -> + def par = config.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") } + assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid input argument" + + value = _checkArgumentType("input", par, value, "in module '$key' id '$id'") + + [ name, value ] + } + } + return inputs +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_processOutputValues.nf' +Map _processOutputValues(Map outputs, Map config, String id, String key) { + if (!workflow.stubRun) { + config.allArguments.each { arg -> + if (arg.direction == "output" && arg.required) { + assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : + "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing" + } + } + + outputs = outputs.collectEntries { name, value -> + def par = config.allArguments.find { it.plainName == name && it.direction == "output" } + assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid output argument" + + value = _checkArgumentType("output", par, value, "in module '$key' id '$id'") + + [ name, value ] + } + } + return outputs +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/IDChecker.nf' +class IDChecker { + final def items = [] as Set + + @groovy.transform.WithWriteLock + boolean observe(String item) { + if (items.contains(item)) { + return false + } else { + items << item + return true + } + } + + @groovy.transform.WithReadLock + boolean contains(String item) { + return items.contains(item) + } + + @groovy.transform.WithReadLock + Set getItems() { + return items.clone() + } +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_checkUniqueIds.nf' + +/** + * Check if the ids are unique across parameter sets + * + * @param parameterSets a list of parameter sets. + */ +private void _checkUniqueIds(List>> parameterSets) { + def ppIds = parameterSets.collect{it[0]} + assert ppIds.size() == ppIds.unique().size() : "All argument sets should have unique ids. Detected ids: $ppIds" +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_getChild.nf' + +// helper functions for reading params from file // +def _getChild(parent, child) { + if (child.contains("://") || java.nio.file.Paths.get(child).isAbsolute()) { + child + } else { + def parentAbsolute = java.nio.file.Paths.get(parent).toAbsolutePath().toString() + parentAbsolute.replaceAll('/[^/]*$', "/") + child + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_parseParamList.nf' +/** + * Figure out the param list format based on the file extension + * + * @param param_list A String containing the path to the parameter list file. + * + * @return A String containing the format of the parameter list file. + */ +def _paramListGuessFormat(param_list) { + if (param_list !instanceof String) { + "asis" + } else if (param_list.endsWith(".csv")) { + "csv" + } else if (param_list.endsWith(".json") || param_list.endsWith(".jsn")) { + "json" + } else if (param_list.endsWith(".yaml") || param_list.endsWith(".yml")) { + "yaml" + } else { + "yaml_blob" + } +} + + +/** + * Read the param list + * + * @param param_list One of the following: + * - A String containing the path to the parameter list file (csv, json or yaml), + * - A yaml blob of a list of maps (yaml_blob), + * - Or a groovy list of maps (asis). + * @param config A Map of the Viash configuration. + * + * @return A List of Maps containing the parameters. + */ +def _parseParamList(param_list, Map config) { + // first determine format by extension + def paramListFormat = _paramListGuessFormat(param_list) + + def paramListPath = (paramListFormat != "asis" && paramListFormat != "yaml_blob") ? + file(param_list, hidden: true) : + null + + // get the correct parser function for the detected params_list format + def paramSets = [] + if (paramListFormat == "asis") { + paramSets = param_list + } else if (paramListFormat == "yaml_blob") { + paramSets = readYamlBlob(param_list) + } else if (paramListFormat == "yaml") { + paramSets = readYaml(paramListPath) + } else if (paramListFormat == "json") { + paramSets = readJson(paramListPath) + } else if (paramListFormat == "csv") { + paramSets = readCsv(paramListPath) + } else { + error "Format of provided --param_list not recognised.\n" + + "Found: '$paramListFormat'.\n" + + "Expected: a csv file, a json file, a yaml file,\n" + + "a yaml blob or a groovy list of maps." + } + + // data checks + assert paramSets instanceof List: "--param_list should contain a list of maps" + for (value in paramSets) { + assert value instanceof Map: "--param_list should contain a list of maps" + } + + // id is argument + def idIsArgument = config.allArguments.any{it.plainName == "id"} + + // Reformat from List to List> by adding the ID as first element of a Tuple2 + paramSets = paramSets.collect({ data -> + def id = data.id + if (!idIsArgument) { + data = data.findAll{k, v -> k != "id"} + } + [id, data] + }) + + // Split parameters with 'multiple: true' + paramSets = paramSets.collect({ id, data -> + data = _splitParams(data, config) + [id, data] + }) + + // The paths of input files inside a param_list file may have been specified relatively to the + // location of the param_list file. These paths must be made absolute. + if (paramListPath) { + paramSets = paramSets.collect({ id, data -> + def new_data = data.collectEntries{ parName, parValue -> + def par = config.allArguments.find{it.plainName == parName} + if (par && par.type == "file" && par.direction == "input") { + if (parValue instanceof Collection) { + parValue = parValue.collectMany{path -> + def x = _resolveSiblingIfNotAbsolute(path, paramListPath) + x instanceof Collection ? x : [x] + } + } else { + parValue = _resolveSiblingIfNotAbsolute(parValue, paramListPath) + } + } + [parName, parValue] + } + [id, new_data] + }) + } + + return paramSets +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_splitParams.nf' +/** + * Split parameters for arguments that accept multiple values using their separator + * + * @param paramList A Map containing parameters to split. + * @param config A Map of the Viash configuration. This Map can be generated from the config file + * using the readConfig() function. + * + * @return A Map of parameters where the parameter values have been split into a list using + * their seperator. + */ +Map _splitParams(Map parValues, Map config){ + def parsedParamValues = parValues.collectEntries { parName, parValue -> + def parameterSettings = config.allArguments.find({it.plainName == parName}) + + if (!parameterSettings) { + // if argument is not found, do not alter + return [parName, parValue] + } + if (parameterSettings.multiple) { // Check if parameter can accept multiple values + if (parValue instanceof Collection) { + parValue = parValue.collect{it instanceof String ? it.split(parameterSettings.multiple_sep) : it } + } else if (parValue instanceof String) { + parValue = parValue.split(parameterSettings.multiple_sep) + } else if (parValue == null) { + parValue = [] + } else { + parValue = [ parValue ] + } + parValue = parValue.flatten() + } + // For all parameters check if multiple values are only passed for + // arguments that allow it. Quietly simplify lists of length 1. + if (!parameterSettings.multiple && parValue instanceof Collection) { + assert parValue.size() == 1 : + "Error: argument ${parName} has too many values.\n" + + " Expected amount: 1. Found: ${parValue.size()}" + parValue = parValue[0] + } + [parName, parValue] + } + return parsedParamValues +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/channelFromParams.nf' +/** + * Parse nextflow parameters based on settings defined in a viash config. + * Return a list of parameter sets, each parameter set corresponding to + * an event in a nextflow channel. The output from this function can be used + * with Channel.fromList to create a nextflow channel with Vdsl3 formatted + * events. + * + * This function performs: + * - A filtering of the params which can be found in the config file. + * - Process the params_list argument which allows a user to to initialise + * a Vsdl3 channel with multiple parameter sets. Possible formats are + * csv, json, yaml, or simply a yaml_blob. A csv should have column names + * which correspond to the different arguments of this pipeline. A json or a yaml + * file should be a list of maps, each of which has keys corresponding to the + * arguments of the pipeline. A yaml blob can also be passed directly as a parameter. + * When passing a csv, json or yaml, relative path names are relativized to the + * location of the parameter file. + * - Combine the parameter sets into a vdsl3 Channel. + * + * @param params Input parameters. Can optionaly contain a 'param_list' key that + * provides a list of arguments that can be split up into multiple events + * in the output channel possible formats of param_lists are: a csv file, + * json file, a yaml file or a yaml blob. Each parameters set (event) must + * have a unique ID. + * @param config A Map of the Viash configuration. This Map can be generated from the config file + * using the readConfig() function. + * + * @return A list of parameters with the first element of the event being + * the event ID and the second element containing a map of the parsed parameters. + */ + +private List>> _paramsToParamSets(Map params, Map config){ + // todo: fetch key from run args + def key_ = config.name + + /* parse regular parameters (not in param_list) */ + /*************************************************/ + def globalParams = config.allArguments + .findAll { params.containsKey(it.plainName) } + .collectEntries { [ it.plainName, params[it.plainName] ] } + def globalID = params.get("id", null) + + /* process params_list arguments */ + /*********************************/ + def paramList = params.containsKey("param_list") && params.param_list != null ? + params.param_list : [] + // if (paramList instanceof String) { + // paramList = [paramList] + // } + // def paramSets = paramList.collectMany{ _parseParamList(it, config) } + // TODO: be able to process param_list when it is a list of strings + def paramSets = _parseParamList(paramList, config) + if (paramSets.isEmpty()) { + paramSets = [[null, [:]]] + } + + /* combine arguments into channel */ + /**********************************/ + def processedParams = paramSets.indexed().collect{ index, tup -> + // Process ID + def id = tup[0] ?: globalID + + if (workflow.stubRun && !id) { + // if stub run, explicitly add an id if missing + id = "stub${index}" + } + assert id != null: "Each parameter set should have at least an 'id'" + + // Process params + def parValues = globalParams + tup[1] + // // Remove parameters which are null, if the default is also null + // parValues = parValues.collectEntries{paramName, paramValue -> + // parameterSettings = config.functionality.allArguments.find({it.plainName == paramName}) + // if ( paramValue != null || parameterSettings.get("default", null) != null ) { + // [paramName, paramValue] + // } + // } + parValues = parValues.collectEntries { name, value -> + def par = config.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") } + assert par != null : "Error in module '${key_}' id '${id}': '${name}' is not a valid input argument" + + if (par == null) { + return [:] + } + value = _checkArgumentType("input", par, value, "in module '$key_' id '$id'") + + [ name, value ] + } + + [id, parValues] + } + + // Check if ids (first element of each list) is unique + _checkUniqueIds(processedParams) + return processedParams +} + +/** + * Parse nextflow parameters based on settings defined in a viash config + * and return a nextflow channel. + * + * @param params Input parameters. Can optionaly contain a 'param_list' key that + * provides a list of arguments that can be split up into multiple events + * in the output channel possible formats of param_lists are: a csv file, + * json file, a yaml file or a yaml blob. Each parameters set (event) must + * have a unique ID. + * @param config A Map of the Viash configuration. This Map can be generated from the config file + * using the readConfig() function. + * + * @return A nextflow Channel with events. Events are formatted as a tuple that contains + * first contains the ID of the event and as second element holds a parameter map. + * + * + */ +def channelFromParams(Map params, Map config) { + def processedParams = _paramsToParamSets(params, config) + return Channel.fromList(processedParams) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/checkUniqueIds.nf' +def checkUniqueIds(Map args) { + def stopOnError = args.stopOnError == null ? args.stopOnError : true + + def idChecker = new IDChecker() + + return filter { tup -> + if (!idChecker.observe(tup[0])) { + if (stopOnError) { + error "Duplicate id: ${tup[0]}" + } else { + log.warn "Duplicate id: ${tup[0]}, removing duplicate entry" + return false + } + } + return true + } +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/preprocessInputs.nf' +// This helper file will be deprecated soon +preprocessInputsDeprecationWarningPrinted = false + +def preprocessInputsDeprecationWarning() { + if (!preprocessInputsDeprecationWarningPrinted) { + preprocessInputsDeprecationWarningPrinted = true + System.err.println("Warning: preprocessInputs() is deprecated and will be removed in Viash 0.9.0.") + } +} + +/** + * Generate a nextflow Workflow that allows processing a channel of + * Vdsl3 formatted events and apply a Viash config to them: + * - Gather default parameters from the Viash config and make + * sure that they are correctly formatted (see applyConfig method). + * - Format the input parameters (also using the applyConfig method). + * - Apply the default parameter to the input parameters. + * - Do some assertions: + * ~ Check if the event IDs in the channel are unique. + * + * The events in the channel are formatted as tuples, with the + * first element of the tuples being a unique id of the parameter set, + * and the second element containg the the parameters themselves. + * Optional extra elements of the tuples will be passed to the output as is. + * + * @param args A map that must contain a 'config' key that points + * to a parsed config (see readConfig()). Optionally, a + * 'key' key can be provided which can be used to create a unique + * name for the workflow process. + * + * @return A workflow that allows processing a channel of Vdsl3 formatted events + * and apply a Viash config to them. + */ +def preprocessInputs(Map args) { + preprocessInputsDeprecationWarning() + + def config = args.config + assert config instanceof Map : + "Error in preprocessInputs: config must be a map. " + + "Expected class: Map. Found: config.getClass() is ${config.getClass()}" + def key_ = args.key ?: config.name + + // Get different parameter types (used throughout this function) + def defaultArgs = config.allArguments + .findAll { it.containsKey("default") } + .collectEntries { [ it.plainName, it.default ] } + + map { tup -> + def id = tup[0] + def data = tup[1] + def passthrough = tup.drop(2) + + def new_data = (defaultArgs + data).collectEntries { name, value -> + def par = config.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") } + + if (par != null) { + value = _checkArgumentType("input", par, value, "in module '$key_' id '$id'") + } + + [ name, value ] + } + + [ id, new_data ] + passthrough + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/runComponents.nf' +/** + * Run a list of components on a stream of data. + * + * @param components: list of Viash VDSL3 modules to run + * @param fromState: a closure, a map or a list of keys to extract from the input data. + * If a closure, it will be called with the id, the data and the component config. + * @param toState: a closure, a map or a list of keys to extract from the output data + * If a closure, it will be called with the id, the output data, the old state and the component config. + * @param filter: filter function to apply to the input. + * It will be called with the id, the data and the component config. + * @param id: id to use for the output data + * If a closure, it will be called with the id, the data and the component config. + * @param auto: auto options to pass to the components + * + * @return: a workflow that runs the components + **/ +def runComponents(Map args) { + log.warn("runComponents is deprecated, use runEach instead") + assert args.components: "runComponents should be passed a list of components to run" + + def components_ = args.components + if (components_ !instanceof List) { + components_ = [ components_ ] + } + assert components_.size() > 0: "pass at least one component to runComponents" + + def fromState_ = args.fromState + def toState_ = args.toState + def filter_ = args.filter + def id_ = args.id + + workflow runComponentsWf { + take: input_ch + main: + + // generate one channel per method + out_chs = components_.collect{ comp_ -> + def comp_config = comp_.config + + def filter_ch = filter_ + ? input_ch | filter{tup -> + filter_(tup[0], tup[1], comp_config) + } + : input_ch + def id_ch = id_ + ? filter_ch | map{tup -> + // def new_id = id_(tup[0], tup[1], comp_config) + def new_id = tup[0] + if (id_ instanceof String) { + new_id = id_ + } else if (id_ instanceof Closure) { + new_id = id_(new_id, tup[1], comp_config) + } + [new_id] + tup.drop(1) + } + : filter_ch + def data_ch = id_ch | map{tup -> + def new_data = tup[1] + if (fromState_ instanceof Map) { + new_data = fromState_.collectEntries{ key0, key1 -> + [key0, new_data[key1]] + } + } else if (fromState_ instanceof List) { + new_data = fromState_.collectEntries{ key -> + [key, new_data[key]] + } + } else if (fromState_ instanceof Closure) { + new_data = fromState_(tup[0], new_data, comp_config) + } + tup.take(1) + [new_data] + tup.drop(1) + } + def out_ch = data_ch + | comp_.run( + auto: (args.auto ?: [:]) + [simplifyInput: false, simplifyOutput: false] + ) + def post_ch = toState_ + ? out_ch | map{tup -> + def output = tup[1] + def old_state = tup[2] + def new_state = null + if (toState_ instanceof Map) { + new_state = old_state + toState_.collectEntries{ key0, key1 -> + [key0, output[key1]] + } + } else if (toState_ instanceof List) { + new_state = old_state + toState_.collectEntries{ key -> + [key, output[key]] + } + } else if (toState_ instanceof Closure) { + new_state = toState_(tup[0], output, old_state, comp_config) + } + [tup[0], new_state] + tup.drop(3) + } + : out_ch + + post_ch + } + + // mix all results + output_ch = + (out_chs.size == 1) + ? out_chs[0] + : out_chs[0].mix(*out_chs.drop(1)) + + emit: output_ch + } + + return runComponentsWf +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/runEach.nf' +/** + * Run a list of components on a stream of data. + * + * @param components: list of Viash VDSL3 modules to run + * @param fromState: a closure, a map or a list of keys to extract from the input data. + * If a closure, it will be called with the id, the data and the component itself. + * @param toState: a closure, a map or a list of keys to extract from the output data + * If a closure, it will be called with the id, the output data, the old state and the component itself. + * @param filter: filter function to apply to the input. + * It will be called with the id, the data and the component itself. + * @param id: id to use for the output data + * If a closure, it will be called with the id, the data and the component itself. + * @param auto: auto options to pass to the components + * + * @return: a workflow that runs the components + **/ +def runEach(Map args) { + assert args.components: "runEach should be passed a list of components to run" + + def components_ = args.components + if (components_ !instanceof List) { + components_ = [ components_ ] + } + assert components_.size() > 0: "pass at least one component to runEach" + + def fromState_ = args.fromState + def toState_ = args.toState + def filter_ = args.filter + def runIf_ = args.runIf + def id_ = args.id + + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + + workflow runEachWf { + take: input_ch + main: + + // generate one channel per method + out_chs = components_.collect{ comp_ -> + def filter_ch = filter_ + ? input_ch | filter{tup -> + filter_(tup[0], tup[1], comp_) + } + : input_ch + def id_ch = id_ + ? filter_ch | map{tup -> + def new_id = id_ + if (new_id instanceof Closure) { + new_id = new_id(tup[0], tup[1], comp_) + } + assert new_id instanceof String : "Error in runEach: id should be a String or a Closure that returns a String. Expected: id instanceof String. Found: ${new_id.getClass()}" + [new_id] + tup.drop(1) + } + : filter_ch + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> + def new_data = tup[1] + if (fromState_ instanceof Map) { + new_data = fromState_.collectEntries{ key0, key1 -> + [key0, new_data[key1]] + } + } else if (fromState_ instanceof List) { + new_data = fromState_.collectEntries{ key -> + [key, new_data[key]] + } + } else if (fromState_ instanceof Closure) { + new_data = fromState_(tup[0], new_data, comp_) + } + tup.take(1) + [new_data] + tup.drop(1) + } + def out_ch = data_ch + | comp_.run( + auto: (args.auto ?: [:]) + [simplifyInput: false, simplifyOutput: false] + ) + def post_ch = toState_ + ? out_ch | map{tup -> + def output = tup[1] + def old_state = tup[2] + def new_state = null + if (toState_ instanceof Map) { + new_state = old_state + toState_.collectEntries{ key0, key1 -> + [key0, output[key1]] + } + } else if (toState_ instanceof List) { + new_state = old_state + toState_.collectEntries{ key -> + [key, output[key]] + } + } else if (toState_ instanceof Closure) { + new_state = toState_(tup[0], output, old_state, comp_) + } + [tup[0], new_state] + tup.drop(3) + } + : out_ch + + def return_ch = post_ch + | concat(chPassthrough) + + return_ch + } + + // mix all results + output_ch = + (out_chs.size == 1) + ? out_chs[0] + : out_chs[0].mix(*out_chs.drop(1)) + + emit: output_ch + } + + return runEachWf +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/safeJoin.nf' +/** + * Join sourceChannel to targetChannel + * + * This function joins the sourceChannel to the targetChannel. + * However, each id in the targetChannel must be present in the + * sourceChannel. If _meta.join_id exists in the targetChannel, that is + * used as an id instead. If the id doesn't match any id in the sourceChannel, + * an error is thrown. + */ + +def safeJoin(targetChannel, sourceChannel, key) { + def sourceIDs = new IDChecker() + + def sourceCheck = sourceChannel + | map { tup -> + sourceIDs.observe(tup[0]) + tup + } + def targetCheck = targetChannel + | map { tup -> + def id = tup[0] + + if (!sourceIDs.contains(id)) { + error ( + "Error in module '${key}' when merging output with original state.\n" + + " Reason: output with id '${id}' could not be joined with source channel.\n" + + " If the IDs in the output channel differ from the input channel,\n" + + " please set `tup[1]._meta.join_id to the original ID.\n" + + " Original IDs in input channel: ['${sourceIDs.getItems().join("', '")}'].\n" + + " Unexpected ID in the output channel: '${id}'.\n" + + " Example input event: [\"id\", [input: file(...)]],\n" + + " Example output event: [\"newid\", [output: file(...), _meta: [join_id: \"id\"]]]" + ) + } + // TODO: add link to our documentation on how to fix this + + tup + } + + sourceCheck.cross(targetChannel) + | map{ left, right -> + right + left.drop(1) + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/config/_processArgument.nf' +def _processArgument(arg) { + arg.multiple = arg.multiple != null ? arg.multiple : false + arg.required = arg.required != null ? arg.required : false + arg.direction = arg.direction != null ? arg.direction : "input" + arg.multiple_sep = arg.multiple_sep != null ? arg.multiple_sep : ";" + arg.plainName = arg.name.replaceAll("^-*", "") + + if (arg.type == "file") { + arg.must_exist = arg.must_exist != null ? arg.must_exist : true + arg.create_parent = arg.create_parent != null ? arg.create_parent : true + } + + // add default values to output files which haven't already got a default + if (arg.type == "file" && arg.direction == "output" && arg.default == null) { + def mult = arg.multiple ? "_*" : "" + def extSearch = "" + if (arg.default != null) { + extSearch = arg.default + } else if (arg.example != null) { + extSearch = arg.example + } + if (extSearch instanceof List) { + extSearch = extSearch[0] + } + def extSearchResult = extSearch.find("\\.[^\\.]+\$") + def ext = extSearchResult != null ? extSearchResult : "" + arg.default = "\$id.\$key.${arg.plainName}${mult}${ext}" + if (arg.multiple) { + arg.default = [arg.default] + } + } + + if (!arg.multiple) { + if (arg.default != null && arg.default instanceof List) { + arg.default = arg.default[0] + } + if (arg.example != null && arg.example instanceof List) { + arg.example = arg.example[0] + } + } + + if (arg.type == "boolean_true") { + arg.default = false + } + if (arg.type == "boolean_false") { + arg.default = true + } + + arg +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/config/addGlobalParams.nf' +def addGlobalArguments(config) { + def localConfig = [ + "argument_groups": [ + [ + "name": "Nextflow input-output arguments", + "description": "Input/output parameters for Nextflow itself. Please note that both publishDir and publish_dir are supported but at least one has to be configured.", + "arguments" : [ + [ + 'name': '--publish_dir', + 'required': true, + 'type': 'string', + 'description': 'Path to an output directory.', + 'example': 'output/', + 'multiple': false + ], + [ + 'name': '--param_list', + 'required': false, + 'type': 'string', + 'description': '''Allows inputting multiple parameter sets to initialise a Nextflow channel. A `param_list` can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. + | + |* A list of maps (as-is) where the keys of each map corresponds to the arguments of the pipeline. Example: in a `nextflow.config` file: `param_list: [ ['id': 'foo', 'input': 'foo.txt'], ['id': 'bar', 'input': 'bar.txt'] ]`. + |* A csv file should have column names which correspond to the different arguments of this pipeline. Example: `--param_list data.csv` with columns `id,input`. + |* A json or a yaml file should be a list of maps, each of which has keys corresponding to the arguments of the pipeline. Example: `--param_list data.json` with contents `[ {'id': 'foo', 'input': 'foo.txt'}, {'id': 'bar', 'input': 'bar.txt'} ]`. + |* A yaml blob can also be passed directly as a string. Example: `--param_list "[ {'id': 'foo', 'input': 'foo.txt'}, {'id': 'bar', 'input': 'bar.txt'} ]"`. + | + |When passing a csv, json or yaml file, relative path names are relativized to the location of the parameter file. No relativation is performed when `param_list` is a list of maps (as-is) or a yaml blob.'''.stripMargin(), + 'example': 'my_params.yaml', + 'multiple': false, + 'hidden': true + ] + // TODO: allow multiple: true in param_list? + // TODO: allow to specify a --param_list_regex to filter the param_list? + // TODO: allow to specify a --param_list_from_state to remap entries in the param_list? + ] + ] + ] + ] + + return processConfig(_mergeMap(config, localConfig)) +} + +def _mergeMap(Map lhs, Map rhs) { + return rhs.inject(lhs.clone()) { map, entry -> + if (map[entry.key] instanceof Map && entry.value instanceof Map) { + map[entry.key] = _mergeMap(map[entry.key], entry.value) + } else if (map[entry.key] instanceof Collection && entry.value instanceof Collection) { + map[entry.key] += entry.value + } else { + map[entry.key] = entry.value + } + return map + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/config/generateHelp.nf' +def _generateArgumentHelp(param) { + // alternatives are not supported + // def names = param.alternatives ::: List(param.name) + + def unnamedProps = [ + ["required parameter", param.required], + ["multiple values allowed", param.multiple], + ["output", param.direction.toLowerCase() == "output"], + ["file must exist", param.type == "file" && param.must_exist] + ].findAll{it[1]}.collect{it[0]} + + def dflt = null + if (param.default != null) { + if (param.default instanceof List) { + dflt = param.default.join(param.multiple_sep != null ? param.multiple_sep : ", ") + } else { + dflt = param.default.toString() + } + } + def example = null + if (param.example != null) { + if (param.example instanceof List) { + example = param.example.join(param.multiple_sep != null ? param.multiple_sep : ", ") + } else { + example = param.example.toString() + } + } + def min = param.min?.toString() + def max = param.max?.toString() + + def escapeChoice = { choice -> + def s1 = choice.replaceAll("\\n", "\\\\n") + def s2 = s1.replaceAll("\"", """\\\"""") + s2.contains(",") || s2 != choice ? "\"" + s2 + "\"" : s2 + } + def choices = param.choices == null ? + null : + "[ " + param.choices.collect{escapeChoice(it.toString())}.join(", ") + " ]" + + def namedPropsStr = [ + ["type", ([param.type] + unnamedProps).join(", ")], + ["default", dflt], + ["example", example], + ["choices", choices], + ["min", min], + ["max", max] + ] + .findAll{it[1]} + .collect{"\n " + it[0] + ": " + it[1].replaceAll("\n", "\\n")} + .join("") + + def descStr = param.description == null ? + "" : + _paragraphWrap("\n" + param.description.trim(), 80 - 8).join("\n ") + + "\n --" + param.plainName + + namedPropsStr + + descStr +} + +// Based on Helper.generateHelp() in Helper.scala +def _generateHelp(config) { + def fun = config + + // PART 1: NAME AND VERSION + def nameStr = fun.name + + (fun.version == null ? "" : " " + fun.version) + + // PART 2: DESCRIPTION + def descrStr = fun.description == null ? + "" : + "\n\n" + _paragraphWrap(fun.description.trim(), 80).join("\n") + + // PART 3: Usage + def usageStr = fun.usage == null ? + "" : + "\n\nUsage:\n" + fun.usage.trim() + + // PART 4: Options + def argGroupStrs = fun.allArgumentGroups.collect{argGroup -> + def name = argGroup.name + def descriptionStr = argGroup.description == null ? + "" : + "\n " + _paragraphWrap(argGroup.description.trim(), 80-4).join("\n ") + "\n" + def arguments = argGroup.arguments.collect{arg -> + arg instanceof String ? fun.allArguments.find{it.plainName == arg} : arg + }.findAll{it != null} + def argumentStrs = arguments.collect{param -> _generateArgumentHelp(param)} + + "\n\n$name:" + + descriptionStr + + argumentStrs.join("\n") + } + + // FINAL: combine + def out = nameStr + + descrStr + + usageStr + + argGroupStrs.join("") + + return out +} + +// based on Format._paragraphWrap +def _paragraphWrap(str, maxLength) { + def outLines = [] + str.split("\n").each{par -> + def words = par.split("\\s").toList() + + def word = null + def line = words.pop() + while(!words.isEmpty()) { + word = words.pop() + if (line.length() + word.length() + 1 <= maxLength) { + line = line + " " + word + } else { + outLines.add(line) + line = word + } + } + if (words.isEmpty()) { + outLines.add(line) + } + } + return outLines +} + +def helpMessage(config) { + if (params.containsKey("help") && params.help) { + def mergedConfig = addGlobalArguments(config) + def helpStr = _generateHelp(mergedConfig) + println(helpStr) + exit 0 + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/config/processConfig.nf' +def processConfig(config) { + // set defaults for arguments + config.arguments = + (config.arguments ?: []).collect{_processArgument(it)} + + // set defaults for argument_group arguments + config.argument_groups = + (config.argument_groups ?: []).collect{grp -> + grp.arguments = (grp.arguments ?: []).collect{_processArgument(it)} + grp + } + + // create combined arguments list + config.allArguments = + config.arguments + + config.argument_groups.collectMany{it.arguments} + + // add missing argument groups (based on Functionality::allArgumentGroups()) + def argGroups = config.argument_groups + if (argGroups.any{it.name.toLowerCase() == "arguments"}) { + argGroups = argGroups.collect{ grp -> + if (grp.name.toLowerCase() == "arguments") { + grp = grp + [ + arguments: grp.arguments + config.arguments + ] + } + grp + } + } else { + argGroups = argGroups + [ + name: "Arguments", + arguments: config.arguments + ] + } + config.allArgumentGroups = argGroups + + config +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/config/readConfig.nf' + +def readConfig(file) { + def config = readYaml(file ?: moduleDir.resolve("config.vsh.yaml")) + processConfig(config) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/_resolveSiblingIfNotAbsolute.nf' +/** + * Resolve a path relative to the current file. + * + * @param str The path to resolve, as a String. + * @param parentPath The path to resolve relative to, as a Path. + * + * @return The path that may have been resovled, as a Path. + */ +def _resolveSiblingIfNotAbsolute(str, parentPath) { + if (str !instanceof String) { + return str + } + if (!_stringIsAbsolutePath(str)) { + return parentPath.resolveSibling(str) + } else { + return file(str, hidden: true) + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/_stringIsAbsolutePath.nf' +/** + * Check whether a path as a string is absolute. + * + * In the past, we tried using `file(., relative: true).isAbsolute()`, + * but the 'relative' option was added in 22.10.0. + * + * @param path The path to check, as a String. + * + * @return Whether the path is absolute, as a boolean. + */ +def _stringIsAbsolutePath(path) { + def _resolve_URL_PROTOCOL = ~/^([a-zA-Z][a-zA-Z0-9]*:)?\\/.+/ + + assert path instanceof String + return _resolve_URL_PROTOCOL.matcher(path).matches() +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/collectTraces.nf' +class CustomTraceObserver implements nextflow.trace.TraceObserver { + List traces + + CustomTraceObserver(List traces) { + this.traces = traces + } + + @Override + void onProcessComplete(nextflow.processor.TaskHandler handler, nextflow.trace.TraceRecord trace) { + def trace2 = trace.store.clone() + trace2.script = null + traces.add(trace2) + } + + @Override + void onProcessCached(nextflow.processor.TaskHandler handler, nextflow.trace.TraceRecord trace) { + def trace2 = trace.store.clone() + trace2.script = null + traces.add(trace2) + } +} + +def collectTraces() { + def traces = Collections.synchronizedList([]) + + // add custom trace observer which stores traces in the traces object + session.observers.add(new CustomTraceObserver(traces)) + + traces +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/deepClone.nf' +/** + * Performs a deep clone of the given object. + * @param x an object + */ +def deepClone(x) { + iterateMap(x, {it instanceof Cloneable ? it.clone() : it}) +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/getPublishDir.nf' +def getPublishDir() { + return params.containsKey("publish_dir") ? params.publish_dir : + params.containsKey("publishDir") ? params.publishDir : + null +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/getRootDir.nf' + +// Recurse upwards until we find a '.build.yaml' file +def _findBuildYamlFile(pathPossiblySymlink) { + def path = pathPossiblySymlink.toRealPath() + def child = path.resolve(".build.yaml") + if (java.nio.file.Files.isDirectory(path) && java.nio.file.Files.exists(child)) { + return child + } else { + def parent = path.getParent() + if (parent == null) { + return null + } else { + return _findBuildYamlFile(parent) + } + } +} + +// get the root of the target folder +def getRootDir() { + def dir = _findBuildYamlFile(meta.resources_dir) + assert dir != null: "Could not find .build.yaml in the folder structure" + dir.getParent() +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/iterateMap.nf' +/** + * Recursively apply a function over the leaves of an object. + * @param obj The object to iterate over. + * @param fun The function to apply to each value. + * @return The object with the function applied to each value. + */ +def iterateMap(obj, fun) { + if (obj instanceof List && obj !instanceof String) { + return obj.collect{item -> + iterateMap(item, fun) + } + } else if (obj instanceof Map) { + return obj.collectEntries{key, item -> + [key.toString(), iterateMap(item, fun)] + } + } else { + return fun(obj) + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/niceView.nf' +/** + * A view for printing the event of each channel as a YAML blob. + * This is useful for debugging. + */ +def niceView() { + workflow niceViewWf { + take: input + main: + output = input + | view{toYamlBlob(it)} + emit: output + } + return niceViewWf +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readCsv.nf' + +def readCsv(file_path) { + def output = [] + def inputFile = file_path !instanceof Path ? file(file_path, hidden: true) : file_path + + // todo: allow escaped quotes in string + // todo: allow single quotes? + def splitRegex = java.util.regex.Pattern.compile(''',(?=(?:[^"]*"[^"]*")*[^"]*$)''') + def removeQuote = java.util.regex.Pattern.compile('''"(.*)"''') + + def br = java.nio.file.Files.newBufferedReader(inputFile) + + def row = -1 + def header = null + while (br.ready() && header == null) { + def line = br.readLine() + row++ + if (!line.startsWith("#")) { + header = splitRegex.split(line, -1).collect{field -> + m = removeQuote.matcher(field) + m.find() ? m.replaceFirst('$1') : field + } + } + } + assert header != null: "CSV file should contain a header" + + while (br.ready()) { + def line = br.readLine() + row++ + if (line == null) { + br.close() + break + } + + if (!line.startsWith("#")) { + def predata = splitRegex.split(line, -1) + def data = predata.collect{field -> + if (field == "") { + return null + } + def m = removeQuote.matcher(field) + if (m.find()) { + return m.replaceFirst('$1') + } else { + return field + } + } + assert header.size() == data.size(): "Row $row should contain the same number as fields as the header" + + def dataMap = [header, data].transpose().collectEntries().findAll{it.value != null} + output.add(dataMap) + } + } + + output +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readJson.nf' +def readJson(file_path) { + def inputFile = file_path !instanceof Path ? file(file_path, hidden: true) : file_path + def jsonSlurper = new groovy.json.JsonSlurper() + jsonSlurper.parse(inputFile) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readJsonBlob.nf' +def readJsonBlob(str) { + def jsonSlurper = new groovy.json.JsonSlurper() + jsonSlurper.parseText(str) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readTaggedYaml.nf' +// Custom constructor to modify how certain objects are parsed from YAML +class CustomConstructor extends org.yaml.snakeyaml.constructor.Constructor { + Path root + + class ConstructPath extends org.yaml.snakeyaml.constructor.AbstractConstruct { + public Object construct(org.yaml.snakeyaml.nodes.Node node) { + String filename = (String) constructScalar(node); + if (root != null) { + return root.resolve(filename); + } + return java.nio.file.Paths.get(filename); + } + } + + CustomConstructor(org.yaml.snakeyaml.LoaderOptions options, Path root) { + super(options) + this.root = root + // Handling !file tag and parse it back to a File type + this.yamlConstructors.put(new org.yaml.snakeyaml.nodes.Tag("!file"), new ConstructPath()) + } +} + +def readTaggedYaml(Path path) { + def options = new org.yaml.snakeyaml.LoaderOptions() + def constructor = new CustomConstructor(options, path.getParent()) + def yaml = new org.yaml.snakeyaml.Yaml(constructor) + return yaml.load(path.text) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readYaml.nf' +def readYaml(file_path) { + def inputFile = file_path !instanceof Path ? file(file_path, hidden: true) : file_path + def yamlSlurper = new org.yaml.snakeyaml.Yaml() + yamlSlurper.load(inputFile) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readYamlBlob.nf' +def readYamlBlob(str) { + def yamlSlurper = new org.yaml.snakeyaml.Yaml() + yamlSlurper.load(str) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/toJsonBlob.nf' +String toJsonBlob(data) { + return groovy.json.JsonOutput.toJson(data) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/toTaggedYamlBlob.nf' +// Custom representer to modify how certain objects are represented in YAML +class CustomRepresenter extends org.yaml.snakeyaml.representer.Representer { + Path relativizer + + class RepresentPath implements org.yaml.snakeyaml.representer.Represent { + public String getFileName(Object obj) { + if (obj instanceof File) { + obj = ((File) obj).toPath(); + } + if (obj !instanceof Path) { + throw new IllegalArgumentException("Object: " + obj + " is not a Path or File"); + } + def path = (Path) obj; + + if (relativizer != null) { + return relativizer.relativize(path).toString() + } else { + return path.toString() + } + } + + public org.yaml.snakeyaml.nodes.Node representData(Object data) { + String filename = getFileName(data); + def tag = new org.yaml.snakeyaml.nodes.Tag("!file"); + return representScalar(tag, filename); + } + } + CustomRepresenter(org.yaml.snakeyaml.DumperOptions options, Path relativizer) { + super(options) + this.relativizer = relativizer + this.representers.put(sun.nio.fs.UnixPath, new RepresentPath()) + this.representers.put(Path, new RepresentPath()) + this.representers.put(File, new RepresentPath()) + } +} + +String toTaggedYamlBlob(data) { + return toRelativeTaggedYamlBlob(data, null) +} +String toRelativeTaggedYamlBlob(data, Path relativizer) { + def options = new org.yaml.snakeyaml.DumperOptions() + options.setDefaultFlowStyle(org.yaml.snakeyaml.DumperOptions.FlowStyle.BLOCK) + def representer = new CustomRepresenter(options, relativizer) + def yaml = new org.yaml.snakeyaml.Yaml(representer, options) + return yaml.dump(data) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/toYamlBlob.nf' +String toYamlBlob(data) { + def options = new org.yaml.snakeyaml.DumperOptions() + options.setDefaultFlowStyle(org.yaml.snakeyaml.DumperOptions.FlowStyle.BLOCK) + options.setPrettyFlow(true) + def yaml = new org.yaml.snakeyaml.Yaml(options) + def cleanData = iterateMap(data, { it instanceof Path ? it.toString() : it }) + return yaml.dump(cleanData) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/writeJson.nf' +void writeJson(data, file) { + assert data: "writeJson: data should not be null" + assert file: "writeJson: file should not be null" + file.write(toJsonBlob(data)) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/writeYaml.nf' +void writeYaml(data, file) { + assert data: "writeYaml: data should not be null" + assert file: "writeYaml: file should not be null" + file.write(toYamlBlob(data)) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/findStates.nf' +def findStates(Map params, Map config) { + def auto_config = deepClone(config) + def auto_params = deepClone(params) + + auto_config = auto_config.clone() + // override arguments + auto_config.argument_groups = [] + auto_config.arguments = [ + [ + type: "string", + name: "--id", + description: "A dummy identifier", + required: false + ], + [ + type: "file", + name: "--input_states", + example: "/path/to/input/directory/**/state.yaml", + description: "Path to input directory containing the datasets to be integrated.", + required: true, + multiple: true, + multiple_sep: ";" + ], + [ + type: "string", + name: "--filter", + example: "foo/.*/state.yaml", + description: "Regex to filter state files by path.", + required: false + ], + // to do: make this a yaml blob? + [ + type: "string", + name: "--rename_keys", + example: ["newKey1:oldKey1", "newKey2:oldKey2"], + description: "Rename keys in the detected input files. This is useful if the input files do not match the set of input arguments of the workflow.", + required: false, + multiple: true, + multiple_sep: ";" + ], + [ + type: "string", + name: "--settings", + example: '{"output_dataset": "dataset.h5ad", "k": 10}', + description: "Global arguments as a JSON glob to be passed to all components.", + required: false + ] + ] + if (!(auto_params.containsKey("id"))) { + auto_params["id"] = "auto" + } + + // run auto config through processConfig once more + auto_config = processConfig(auto_config) + + workflow findStatesWf { + helpMessage(auto_config) + + output_ch = + channelFromParams(auto_params, auto_config) + | flatMap { autoId, args -> + + def globalSettings = args.settings ? readYamlBlob(args.settings) : [:] + + // look for state files in input dir + def stateFiles = args.input_states + + // filter state files by regex + if (args.filter) { + stateFiles = stateFiles.findAll{ stateFile -> + def stateFileStr = stateFile.toString() + def matcher = stateFileStr =~ args.filter + matcher.matches()} + } + + // read in states + def states = stateFiles.collect { stateFile -> + def state_ = readTaggedYaml(stateFile) + [state_.id, state_] + } + + // construct renameMap + if (args.rename_keys) { + def renameMap = args.rename_keys.collectEntries{renameString -> + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" + split + } + + // rename keys in state, only let states through which have all keys + // also add global settings + states = states.collectMany{id, state -> + def newState = [:] + + for (key in renameMap.keySet()) { + def origKey = renameMap[key] + if (!(state.containsKey(origKey))) { + return [] + } + newState[key] = state[origKey] + } + + [[id, globalSettings + newState]] + } + } + + states + } + emit: + output_ch + } + + return findStatesWf +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/joinStates.nf' +def joinStates(Closure apply_) { + workflow joinStatesWf { + take: input_ch + main: + output_ch = input_ch + | toSortedList + | filter{ it.size() > 0 } + | map{ tups -> + def ids = tups.collect{it[0]} + def states = tups.collect{it[1]} + apply_(ids, states) + } + + emit: output_ch + } + return joinStatesWf +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishStates.nf' +def collectFiles(obj) { + if (obj instanceof java.io.File || obj instanceof Path) { + return [obj] + } else if (obj instanceof List && obj !instanceof String) { + return obj.collectMany{item -> + collectFiles(item) + } + } else if (obj instanceof Map) { + return obj.collectMany{key, item -> + collectFiles(item) + } + } else { + return [] + } +} + +/** + * Recurse through a state and collect all input files and their target output filenames. + * @param obj The state to recurse through. + * @param prefix The prefix to prepend to the output filenames. + */ +def collectInputOutputPaths(obj, prefix) { + if (obj instanceof File || obj instanceof Path) { + def path = obj instanceof Path ? obj : obj.toPath() + def ext = path.getFileName().toString().find("\\.[^\\.]+\$") ?: "" + def newFilename = prefix + ext + return [[obj, newFilename]] + } else if (obj instanceof List && obj !instanceof String) { + return obj.withIndex().collectMany{item, ix -> + collectInputOutputPaths(item, prefix + "_" + ix) + } + } else if (obj instanceof Map) { + return obj.collectMany{key, item -> + collectInputOutputPaths(item, prefix + "." + key) + } + } else { + return [] + } +} + +def publishStates(Map args) { + def key_ = args.get("key") + def yamlTemplate_ = args.get("output_state", args.get("outputState", '$id.$key.state.yaml')) + + assert key_ != null : "publishStates: key must be specified" + + workflow publishStatesWf { + take: input_ch + main: + input_ch + | map { tup -> + def id_ = tup[0] + def state_ = tup[1] + + // the input files and the target output filenames + def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose() + def inputFiles_ = inputoutputFilenames_[0] + def outputFilenames_ = inputoutputFilenames_[1] + + def yamlFilename = yamlTemplate_ + .replaceAll('\\$id', id_) + .replaceAll('\\$key', key_) + + // TODO: do the pathnames in state_ match up with the outputFilenames_? + + // convert state to yaml blob + def yamlBlob_ = toRelativeTaggedYamlBlob([id: id_] + state_, java.nio.file.Paths.get(yamlFilename)) + + [id_, yamlBlob_, yamlFilename, inputFiles_, outputFilenames_] + } + | publishStatesProc + emit: input_ch + } + return publishStatesWf +} +process publishStatesProc { + // todo: check publishpath? + publishDir path: "${getPublishDir()}/", mode: "copy" + tag "$id" + input: + tuple val(id), val(yamlBlob), val(yamlFile), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles) + output: + tuple val(id), path{[yamlFile] + outputFiles} + script: + def copyCommands = [ + inputFiles instanceof List ? inputFiles : [inputFiles], + outputFiles instanceof List ? outputFiles : [outputFiles] + ] + .transpose() + .collectMany{infile, outfile -> + if (infile.toString() != outfile.toString()) { + [ + "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"", + "cp -r '${infile.toString()}' '${outfile.toString()}'" + ] + } else { + // no need to copy if infile is the same as outfile + [] + } + } + """ +mkdir -p "\$(dirname '${yamlFile}')" +echo "Storing state as yaml" +echo '${yamlBlob}' > '${yamlFile}' +echo "Copying output files to destination folder" +${copyCommands.join("\n ")} +""" +} + + +// this assumes that the state contains no other values other than those specified in the config +def publishStatesByConfig(Map args) { + def config = args.get("config") + assert config != null : "publishStatesByConfig: config must be specified" + + def key_ = args.get("key", config.name) + assert key_ != null : "publishStatesByConfig: key must be specified" + + workflow publishStatesSimpleWf { + take: input_ch + main: + input_ch + | map { tup -> + def id_ = tup[0] + def state_ = tup[1] // e.g. [output: new File("myoutput.h5ad"), k: 10] + def origState_ = tup[2] // e.g. [output: '$id.$key.foo.h5ad'] + + // TODO: allow overriding the state.yaml template + // TODO TODO: if auto.publish == "state", add output_state as an argument + def yamlTemplate = params.containsKey("output_state") ? params.output_state : '$id.$key.state.yaml' + def yamlFilename = yamlTemplate + .replaceAll('\\$id', id_) + .replaceAll('\\$key', key_) + def yamlDir = java.nio.file.Paths.get(yamlFilename).getParent() + + // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where + // - key is a String + // - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path) + // - inputPath is a List[Path] + // - outputFilename is a List[String] + // - (key, value) are the tuples that will be saved to the state.yaml file + // - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml) + def processedState = + config.allArguments + .findAll { it.direction == "output" } + .collectMany { par -> + def plainName_ = par.plainName + // if the state does not contain the key, it's an + // optional argument for which the component did + // not generate any output + if (!state_.containsKey(plainName_)) { + return [] + } + def value = state_[plainName_] + // if the parameter is not a file, it should be stored + // in the state as-is, but is not something that needs + // to be copied from the source path to the dest path + if (par.type != "file") { + return [[key: plainName_, value: value, inputPath: [], outputFilename: []]] + } + // if the orig state does not contain this filename, + // it's an optional argument for which the user specified + // that it should not be returned as a state + if (!origState_.containsKey(plainName_)) { + return [] + } + def filenameTemplate = origState_[plainName_] + // if the pararameter is multiple: true, fetch the template + if (par.multiple && filenameTemplate instanceof List) { + filenameTemplate = filenameTemplate[0] + } + // instantiate the template + def filename = filenameTemplate + .replaceAll('\\$id', id_) + .replaceAll('\\$key', key_) + if (par.multiple) { + // if the parameter is multiple: true, the filename + // should contain a wildcard '*' that is replaced with + // the index of the file + assert filename.contains("*") : "Module '${key_}' id '${id_}': Multiple output files specified, but no wildcard '*' in the filename: ${filename}" + def outputPerFile = value.withIndex().collect{ val, ix -> + def filename_ix = filename.replace("*", ix.toString()) + def value_ = java.nio.file.Paths.get(filename_ix) + // if id contains a slash + if (yamlDir != null) { + value_ = yamlDir.relativize(value_) + } + def inputPath = val instanceof File ? val.toPath() : val + [value: value_, inputPath: inputPath, outputFilename: filename_ix] + } + def transposedOutputs = ["value", "inputPath", "outputFilename"].collectEntries{ key -> + [key, outputPerFile.collect{dic -> dic[key]}] + } + return [[key: plainName_] + transposedOutputs] + } else { + def value_ = java.nio.file.Paths.get(filename) + // if id contains a slash + if (yamlDir != null) { + value_ = yamlDir.relativize(value_) + } + def inputPath = value instanceof File ? value.toPath() : value + return [[key: plainName_, value: value_, inputPath: [inputPath], outputFilename: [filename]]] + } + } + + def updatedState_ = processedState.collectEntries{[it.key, it.value]} + def inputPaths = processedState.collectMany{it.inputPath} + def outputFilenames = processedState.collectMany{it.outputFilename} + + // convert state to yaml blob + def yamlBlob_ = toTaggedYamlBlob([id: id_] + updatedState_) + + [id_, yamlBlob_, yamlFilename, inputPaths, outputFilenames] + } + | publishStatesProc + emit: input_ch + } + return publishStatesSimpleWf +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/setState.nf' +def setState(fun) { + assert fun instanceof Closure || fun instanceof Map || fun instanceof List : + "Error in setState: Expected process argument to be a Closure, a Map, or a List. Found: class ${fun.getClass()}" + + // if fun is a List, convert to map + if (fun instanceof List) { + // check whether fun is a list[string] + assert fun.every{it instanceof CharSequence} : "Error in setState: argument is a List, but not all elements are Strings" + fun = fun.collectEntries{[it, it]} + } + + // if fun is a map, convert to closure + if (fun instanceof Map) { + // check whether fun is a map[string, string] + assert fun.values().every{it instanceof CharSequence} : "Error in setState: argument is a Map, but not all values are Strings" + assert fun.keySet().every{it instanceof CharSequence} : "Error in setState: argument is a Map, but not all keys are Strings" + def funMap = fun.clone() + // turn the map into a closure to be used later on + fun = { id_, state_ -> + assert state_ instanceof Map : "Error in setState: the state is not a Map" + funMap.collectMany{newkey, origkey -> + if (state_.containsKey(origkey)) { + [[newkey, state_[origkey]]] + } else { + [] + } + }.collectEntries() + } + } + + map { tup -> + def id = tup[0] + def state = tup[1] + def unfilteredState = fun(id, state) + def newState = unfilteredState.findAll{key, val -> val != null} + [id, newState] + tup.drop(2) + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/processAuto.nf' +// TODO: unit test processAuto +def processAuto(Map auto) { + // remove null values + auto = auto.findAll{k, v -> v != null} + + // check for unexpected keys + def expectedKeys = ["simplifyInput", "simplifyOutput", "transcript", "publish"] + def unexpectedKeys = auto.keySet() - expectedKeys + assert unexpectedKeys.isEmpty(), "unexpected keys in auto: '${unexpectedKeys.join("', '")}'" + + // check auto.simplifyInput + assert auto.simplifyInput instanceof Boolean, "auto.simplifyInput must be a boolean" + + // check auto.simplifyOutput + assert auto.simplifyOutput instanceof Boolean, "auto.simplifyOutput must be a boolean" + + // check auto.transcript + assert auto.transcript instanceof Boolean, "auto.transcript must be a boolean" + + // check auto.publish + assert auto.publish instanceof Boolean || auto.publish == "state", "auto.publish must be a boolean or 'state'" + + return auto.subMap(expectedKeys) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/processDirectives.nf' +def assertMapKeys(map, expectedKeys, requiredKeys, mapName) { + assert map instanceof Map : "Expected argument '$mapName' to be a Map. Found: class ${map.getClass()}" + map.forEach { key, val -> + assert key in expectedKeys : "Unexpected key '$key' in ${mapName ? mapName + " " : ""}map" + } + requiredKeys.forEach { requiredKey -> + assert map.containsKey(requiredKey) : "Missing required key '$key' in ${mapName ? mapName + " " : ""}map" + } +} + +// TODO: unit test processDirectives +def processDirectives(Map drctv) { + // remove null values + drctv = drctv.findAll{k, v -> v != null} + + // check for unexpected keys + def expectedKeys = [ + "accelerator", "afterScript", "beforeScript", "cache", "conda", "container", "containerOptions", "cpus", "disk", "echo", "errorStrategy", "executor", "machineType", "maxErrors", "maxForks", "maxRetries", "memory", "module", "penv", "pod", "publishDir", "queue", "label", "scratch", "storeDir", "stageInMode", "stageOutMode", "tag", "time" + ] + def unexpectedKeys = drctv.keySet() - expectedKeys + assert unexpectedKeys.isEmpty() : "Unexpected keys in process directive: '${unexpectedKeys.join("', '")}'" + + /* DIRECTIVE accelerator + accepted examples: + - [ limit: 4, type: "nvidia-tesla-k80" ] + */ + if (drctv.containsKey("accelerator")) { + assertMapKeys(drctv["accelerator"], ["type", "limit", "request", "runtime"], [], "accelerator") + } + + /* DIRECTIVE afterScript + accepted examples: + - "source /cluster/bin/cleanup" + */ + if (drctv.containsKey("afterScript")) { + assert drctv["afterScript"] instanceof CharSequence + } + + /* DIRECTIVE beforeScript + accepted examples: + - "source /cluster/bin/setup" + */ + if (drctv.containsKey("beforeScript")) { + assert drctv["beforeScript"] instanceof CharSequence + } + + /* DIRECTIVE cache + accepted examples: + - true + - false + - "deep" + - "lenient" + */ + if (drctv.containsKey("cache")) { + assert drctv["cache"] instanceof CharSequence || drctv["cache"] instanceof Boolean + if (drctv["cache"] instanceof CharSequence) { + assert drctv["cache"] in ["deep", "lenient"] : "Unexpected value for cache" + } + } + + /* DIRECTIVE conda + accepted examples: + - "bwa=0.7.15" + - "bwa=0.7.15 fastqc=0.11.5" + - ["bwa=0.7.15", "fastqc=0.11.5"] + */ + if (drctv.containsKey("conda")) { + if (drctv["conda"] instanceof List) { + drctv["conda"] = drctv["conda"].join(" ") + } + assert drctv["conda"] instanceof CharSequence + } + + /* DIRECTIVE container + accepted examples: + - "foo/bar:tag" + - [ registry: "reg", image: "im", tag: "ta" ] + is transformed to "reg/im:ta" + - [ image: "im" ] + is transformed to "im:latest" + */ + if (drctv.containsKey("container")) { + assert drctv["container"] instanceof Map || drctv["container"] instanceof CharSequence + if (drctv["container"] instanceof Map) { + def m = drctv["container"] + assertMapKeys(m, [ "registry", "image", "tag" ], ["image"], "container") + def part1 = + System.getenv('OVERRIDE_CONTAINER_REGISTRY') ? System.getenv('OVERRIDE_CONTAINER_REGISTRY') + "/" : + params.containsKey("override_container_registry") ? params["override_container_registry"] + "/" : // todo: remove? + m.registry ? m.registry + "/" : + "" + def part2 = m.image + def part3 = m.tag ? ":" + m.tag : ":latest" + drctv["container"] = part1 + part2 + part3 + } + } + + /* DIRECTIVE containerOptions + accepted examples: + - "--foo bar" + - ["--foo bar", "-f b"] + */ + if (drctv.containsKey("containerOptions")) { + if (drctv["containerOptions"] instanceof List) { + drctv["containerOptions"] = drctv["containerOptions"].join(" ") + } + assert drctv["containerOptions"] instanceof CharSequence + } + + /* DIRECTIVE cpus + accepted examples: + - 1 + - 10 + */ + if (drctv.containsKey("cpus")) { + assert drctv["cpus"] instanceof Integer + } + + /* DIRECTIVE disk + accepted examples: + - "1 GB" + - "2TB" + - "3.2KB" + - "10.B" + */ + if (drctv.containsKey("disk")) { + assert drctv["disk"] instanceof CharSequence + // assert drctv["disk"].matches("[0-9]+(\\.[0-9]*)? *[KMGTPEZY]?B") + // ^ does not allow closures + } + + /* DIRECTIVE echo + accepted examples: + - true + - false + */ + if (drctv.containsKey("echo")) { + assert drctv["echo"] instanceof Boolean + } + + /* DIRECTIVE errorStrategy + accepted examples: + - "terminate" + - "finish" + */ + if (drctv.containsKey("errorStrategy")) { + assert drctv["errorStrategy"] instanceof CharSequence + assert drctv["errorStrategy"] in ["terminate", "finish", "ignore", "retry"] : "Unexpected value for errorStrategy" + } + + /* DIRECTIVE executor + accepted examples: + - "local" + - "sge" + */ + if (drctv.containsKey("executor")) { + assert drctv["executor"] instanceof CharSequence + assert drctv["executor"] in ["local", "sge", "uge", "lsf", "slurm", "pbs", "pbspro", "moab", "condor", "nqsii", "ignite", "k8s", "awsbatch", "google-pipelines"] : "Unexpected value for executor" + } + + /* DIRECTIVE machineType + accepted examples: + - "n1-highmem-8" + */ + if (drctv.containsKey("machineType")) { + assert drctv["machineType"] instanceof CharSequence + } + + /* DIRECTIVE maxErrors + accepted examples: + - 1 + - 3 + */ + if (drctv.containsKey("maxErrors")) { + assert drctv["maxErrors"] instanceof Integer + } + + /* DIRECTIVE maxForks + accepted examples: + - 1 + - 3 + */ + if (drctv.containsKey("maxForks")) { + assert drctv["maxForks"] instanceof Integer + } + + /* DIRECTIVE maxRetries + accepted examples: + - 1 + - 3 + */ + if (drctv.containsKey("maxRetries")) { + assert drctv["maxRetries"] instanceof Integer + } + + /* DIRECTIVE memory + accepted examples: + - "1 GB" + - "2TB" + - "3.2KB" + - "10.B" + */ + if (drctv.containsKey("memory")) { + assert drctv["memory"] instanceof CharSequence + // assert drctv["memory"].matches("[0-9]+(\\.[0-9]*)? *[KMGTPEZY]?B") + // ^ does not allow closures + } + + /* DIRECTIVE module + accepted examples: + - "ncbi-blast/2.2.27" + - "ncbi-blast/2.2.27:t_coffee/10.0" + - ["ncbi-blast/2.2.27", "t_coffee/10.0"] + */ + if (drctv.containsKey("module")) { + if (drctv["module"] instanceof List) { + drctv["module"] = drctv["module"].join(":") + } + assert drctv["module"] instanceof CharSequence + } + + /* DIRECTIVE penv + accepted examples: + - "smp" + */ + if (drctv.containsKey("penv")) { + assert drctv["penv"] instanceof CharSequence + } + + /* DIRECTIVE pod + accepted examples: + - [ label: "key", value: "val" ] + - [ annotation: "key", value: "val" ] + - [ env: "key", value: "val" ] + - [ [label: "l", value: "v"], [env: "e", value: "v"]] + */ + if (drctv.containsKey("pod")) { + if (drctv["pod"] instanceof Map) { + drctv["pod"] = [ drctv["pod"] ] + } + assert drctv["pod"] instanceof List + drctv["pod"].forEach { pod -> + assert pod instanceof Map + // TODO: should more checks be added? + // See https://www.nextflow.io/docs/latest/process.html?highlight=directives#pod + // e.g. does it contain 'label' and 'value', or 'annotation' and 'value', or ...? + } + } + + /* DIRECTIVE publishDir + accepted examples: + - [] + - [ [ path: "foo", enabled: true ], [ path: "bar", enabled: false ] ] + - "/path/to/dir" + is transformed to [[ path: "/path/to/dir" ]] + - [ path: "/path/to/dir", mode: "cache" ] + is transformed to [[ path: "/path/to/dir", mode: "cache" ]] + */ + // TODO: should we also look at params["publishDir"]? + if (drctv.containsKey("publishDir")) { + def pblsh = drctv["publishDir"] + + // check different options + assert pblsh instanceof List || pblsh instanceof Map || pblsh instanceof CharSequence + + // turn into list if not already so + // for some reason, 'if (!pblsh instanceof List) pblsh = [ pblsh ]' doesn't work. + pblsh = pblsh instanceof List ? pblsh : [ pblsh ] + + // check elements of publishDir + pblsh = pblsh.collect{ elem -> + // turn into map if not already so + elem = elem instanceof CharSequence ? [ path: elem ] : elem + + // check types and keys + assert elem instanceof Map : "Expected publish argument '$elem' to be a String or a Map. Found: class ${elem.getClass()}" + assertMapKeys(elem, [ "path", "mode", "overwrite", "pattern", "saveAs", "enabled" ], ["path"], "publishDir") + + // check elements in map + assert elem.containsKey("path") + assert elem["path"] instanceof CharSequence + if (elem.containsKey("mode")) { + assert elem["mode"] instanceof CharSequence + assert elem["mode"] in [ "symlink", "rellink", "link", "copy", "copyNoFollow", "move" ] + } + if (elem.containsKey("overwrite")) { + assert elem["overwrite"] instanceof Boolean + } + if (elem.containsKey("pattern")) { + assert elem["pattern"] instanceof CharSequence + } + if (elem.containsKey("saveAs")) { + assert elem["saveAs"] instanceof CharSequence //: "saveAs as a Closure is currently not supported. Surround your closure with single quotes to get the desired effect. Example: '\{ foo \}'" + } + if (elem.containsKey("enabled")) { + assert elem["enabled"] instanceof Boolean + } + + // return final result + elem + } + // store final directive + drctv["publishDir"] = pblsh + } + + /* DIRECTIVE queue + accepted examples: + - "long" + - "short,long" + - ["short", "long"] + */ + if (drctv.containsKey("queue")) { + if (drctv["queue"] instanceof List) { + drctv["queue"] = drctv["queue"].join(",") + } + assert drctv["queue"] instanceof CharSequence + } + + /* DIRECTIVE label + accepted examples: + - "big_mem" + - "big_cpu" + - ["big_mem", "big_cpu"] + */ + if (drctv.containsKey("label")) { + if (drctv["label"] instanceof CharSequence) { + drctv["label"] = [ drctv["label"] ] + } + assert drctv["label"] instanceof List + drctv["label"].forEach { label -> + assert label instanceof CharSequence + // assert label.matches("[a-zA-Z0-9]([a-zA-Z0-9_]*[a-zA-Z0-9])?") + // ^ does not allow closures + } + } + + /* DIRECTIVE scratch + accepted examples: + - true + - "/path/to/scratch" + - '$MY_PATH_TO_SCRATCH' + - "ram-disk" + */ + if (drctv.containsKey("scratch")) { + assert drctv["scratch"] == true || drctv["scratch"] instanceof CharSequence + } + + /* DIRECTIVE storeDir + accepted examples: + - "/path/to/storeDir" + */ + if (drctv.containsKey("storeDir")) { + assert drctv["storeDir"] instanceof CharSequence + } + + /* DIRECTIVE stageInMode + accepted examples: + - "copy" + - "link" + */ + if (drctv.containsKey("stageInMode")) { + assert drctv["stageInMode"] instanceof CharSequence + assert drctv["stageInMode"] in ["copy", "link", "symlink", "rellink"] + } + + /* DIRECTIVE stageOutMode + accepted examples: + - "copy" + - "link" + */ + if (drctv.containsKey("stageOutMode")) { + assert drctv["stageOutMode"] instanceof CharSequence + assert drctv["stageOutMode"] in ["copy", "move", "rsync"] + } + + /* DIRECTIVE tag + accepted examples: + - "foo" + - '$id' + */ + if (drctv.containsKey("tag")) { + assert drctv["tag"] instanceof CharSequence + } + + /* DIRECTIVE time + accepted examples: + - "1h" + - "2days" + - "1day 6hours 3minutes 30seconds" + */ + if (drctv.containsKey("time")) { + assert drctv["time"] instanceof CharSequence + // todo: validation regex? + } + + return drctv +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/processWorkflowArgs.nf' +def processWorkflowArgs(Map args, Map defaultWfArgs, Map meta) { + // override defaults with args + def workflowArgs = defaultWfArgs + args + + // check whether 'key' exists + assert workflowArgs.containsKey("key") : "Error in module '${meta.config.name}': key is a required argument" + + // if 'key' is a closure, apply it to the original key + if (workflowArgs["key"] instanceof Closure) { + workflowArgs["key"] = workflowArgs["key"](meta.config.name) + } + def key = workflowArgs["key"] + assert key instanceof CharSequence : "Expected process argument 'key' to be a String. Found: class ${key.getClass()}" + assert key ==~ /^[a-zA-Z_]\w*$/ : "Error in module '$key': Expected process argument 'key' to consist of only letters, digits or underscores. Found: ${key}" + + // check for any unexpected keys + def expectedKeys = ["key", "directives", "auto", "map", "mapId", "mapData", "mapPassthrough", "filter", "runIf", "fromState", "toState", "args", "renameKeys", "debug"] + def unexpectedKeys = workflowArgs.keySet() - expectedKeys + assert unexpectedKeys.isEmpty() : "Error in module '$key': unexpected arguments to the '.run()' function: '${unexpectedKeys.join("', '")}'" + + // check whether directives exists and apply defaults + assert workflowArgs.containsKey("directives") : "Error in module '$key': directives is a required argument" + assert workflowArgs["directives"] instanceof Map : "Error in module '$key': Expected process argument 'directives' to be a Map. Found: class ${workflowArgs['directives'].getClass()}" + workflowArgs["directives"] = processDirectives(defaultWfArgs.directives + workflowArgs["directives"]) + + // check whether directives exists and apply defaults + assert workflowArgs.containsKey("auto") : "Error in module '$key': auto is a required argument" + assert workflowArgs["auto"] instanceof Map : "Error in module '$key': Expected process argument 'auto' to be a Map. Found: class ${workflowArgs['auto'].getClass()}" + workflowArgs["auto"] = processAuto(defaultWfArgs.auto + workflowArgs["auto"]) + + // auto define publish, if so desired + if (workflowArgs.auto.publish == true && (workflowArgs.directives.publishDir != null ? workflowArgs.directives.publishDir : [:]).isEmpty()) { + // can't assert at this level thanks to the no_publish profile + // assert params.containsKey("publishDir") || params.containsKey("publish_dir") : + // "Error in module '${workflowArgs['key']}': if auto.publish is true, params.publish_dir needs to be defined.\n" + + // " Example: params.publish_dir = \"./output/\"" + def publishDir = getPublishDir() + + if (publishDir != null) { + workflowArgs.directives.publishDir = [[ + path: publishDir, + saveAs: "{ it.startsWith('.') ? null : it }", // don't publish hidden files, by default + mode: "copy" + ]] + } + } + + // auto define transcript, if so desired + if (workflowArgs.auto.transcript == true) { + // can't assert at this level thanks to the no_publish profile + // assert params.containsKey("transcriptsDir") || params.containsKey("transcripts_dir") || params.containsKey("publishDir") || params.containsKey("publish_dir") : + // "Error in module '${workflowArgs['key']}': if auto.transcript is true, either params.transcripts_dir or params.publish_dir needs to be defined.\n" + + // " Example: params.transcripts_dir = \"./transcripts/\"" + def transcriptsDir = + params.containsKey("transcripts_dir") ? params.transcripts_dir : + params.containsKey("transcriptsDir") ? params.transcriptsDir : + params.containsKey("publish_dir") ? params.publish_dir + "/_transcripts" : + params.containsKey("publishDir") ? params.publishDir + "/_transcripts" : + null + if (transcriptsDir != null) { + def timestamp = nextflow.Nextflow.getSession().getWorkflowMetadata().start.format('yyyy-MM-dd_HH-mm-ss') + def transcriptsPublishDir = [ + path: "$transcriptsDir/$timestamp/\${task.process.replaceAll(':', '-')}/\${id}/", + saveAs: "{ it.startsWith('.') ? it.replaceAll('^.', '') : null }", + mode: "copy" + ] + def publishDirs = workflowArgs.directives.publishDir != null ? workflowArgs.directives.publishDir : null ? workflowArgs.directives.publishDir : [] + workflowArgs.directives.publishDir = publishDirs + transcriptsPublishDir + } + } + + // if this is a stubrun, remove certain directives? + if (workflow.stubRun) { + workflowArgs.directives.keySet().removeAll(["publishDir", "cpus", "memory", "label"]) + } + + for (nam in ["map", "mapId", "mapData", "mapPassthrough", "filter", "runIf"]) { + if (workflowArgs.containsKey(nam) && workflowArgs[nam]) { + assert workflowArgs[nam] instanceof Closure : "Error in module '$key': Expected process argument '$nam' to be null or a Closure. Found: class ${workflowArgs[nam].getClass()}" + } + } + + // TODO: should functions like 'map', 'mapId', 'mapData', 'mapPassthrough' be deprecated as well? + for (nam in ["map", "mapData", "mapPassthrough", "renameKeys"]) { + if (workflowArgs.containsKey(nam) && workflowArgs[nam] != null) { + log.warn "module '$key': workflow argument '$nam' is deprecated and will be removed in Viash 0.9.0. Please use 'fromState' and 'toState' instead." + } + } + + // check fromState + workflowArgs["fromState"] = _processFromState(workflowArgs.get("fromState"), key, meta.config) + + // check toState + workflowArgs["toState"] = _processToState(workflowArgs.get("toState"), key, meta.config) + + // return output + return workflowArgs +} + +def _processFromState(fromState, key_, config_) { + assert fromState == null || fromState instanceof Closure || fromState instanceof Map || fromState instanceof List : + "Error in module '$key_': Expected process argument 'fromState' to be null, a Closure, a Map, or a List. Found: class ${fromState.getClass()}" + if (fromState == null) { + return null + } + + // if fromState is a List, convert to map + if (fromState instanceof List) { + // check whether fromstate is a list[string] + assert fromState.every{it instanceof CharSequence} : "Error in module '$key_': fromState is a List, but not all elements are Strings" + fromState = fromState.collectEntries{[it, it]} + } + + // if fromState is a map, convert to closure + if (fromState instanceof Map) { + // check whether fromstate is a map[string, string] + assert fromState.values().every{it instanceof CharSequence} : "Error in module '$key_': fromState is a Map, but not all values are Strings" + assert fromState.keySet().every{it instanceof CharSequence} : "Error in module '$key_': fromState is a Map, but not all keys are Strings" + def fromStateMap = fromState.clone() + def requiredInputNames = meta.config.allArguments.findAll{it.required && it.direction == "Input"}.collect{it.plainName} + // turn the map into a closure to be used later on + fromState = { it -> + def state = it[1] + assert state instanceof Map : "Error in module '$key_': the state is not a Map" + def data = fromStateMap.collectMany{newkey, origkey -> + // check whether newkey corresponds to a required argument + if (state.containsKey(origkey)) { + [[newkey, state[origkey]]] + } else if (!requiredInputNames.contains(origkey)) { + [] + } else { + throw new Exception("Error in module '$key_': fromState key '$origkey' not found in current state") + } + }.collectEntries() + data + } + } + + return fromState +} + +def _processToState(toState, key_, config_) { + if (toState == null) { + toState = { tup -> tup[1] } + } + + // toState should be a closure, map[string, string], or list[string] + assert toState instanceof Closure || toState instanceof Map || toState instanceof List : + "Error in module '$key_': Expected process argument 'toState' to be a Closure, a Map, or a List. Found: class ${toState.getClass()}" + + // if toState is a List, convert to map + if (toState instanceof List) { + // check whether toState is a list[string] + assert toState.every{it instanceof CharSequence} : "Error in module '$key_': toState is a List, but not all elements are Strings" + toState = toState.collectEntries{[it, it]} + } + + // if toState is a map, convert to closure + if (toState instanceof Map) { + // check whether toState is a map[string, string] + assert toState.values().every{it instanceof CharSequence} : "Error in module '$key_': toState is a Map, but not all values are Strings" + assert toState.keySet().every{it instanceof CharSequence} : "Error in module '$key_': toState is a Map, but not all keys are Strings" + def toStateMap = toState.clone() + def requiredOutputNames = config_.allArguments.findAll{it.required && it.direction == "Output"}.collect{it.plainName} + // turn the map into a closure to be used later on + toState = { it -> + def output = it[1] + def state = it[2] + assert output instanceof Map : "Error in module '$key_': the output is not a Map" + assert state instanceof Map : "Error in module '$key_': the state is not a Map" + def extraEntries = toStateMap.collectMany{newkey, origkey -> + // check whether newkey corresponds to a required argument + if (output.containsKey(origkey)) { + [[newkey, output[origkey]]] + } else if (!requiredOutputNames.contains(origkey)) { + [] + } else { + throw new Exception("Error in module '$key_': toState key '$origkey' not found in current output") + } + }.collectEntries() + state + extraEntries + } + } + + return toState +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/workflowFactory.nf' +def _debug(workflowArgs, debugKey) { + if (workflowArgs.debug) { + view { "process '${workflowArgs.key}' $debugKey tuple: $it" } + } else { + map { it } + } +} + +// depends on: innerWorkflowFactory +def workflowFactory(Map args, Map defaultWfArgs, Map meta) { + def workflowArgs = processWorkflowArgs(args, defaultWfArgs, meta) + def key_ = workflowArgs["key"] + + workflow workflowInstance { + take: input_ + + main: + def chModified = input_ + | checkUniqueIds([:]) + | _debug(workflowArgs, "input") + | map { tuple -> + tuple = deepClone(tuple) + + if (workflowArgs.map) { + tuple = workflowArgs.map(tuple) + } + if (workflowArgs.mapId) { + tuple[0] = workflowArgs.mapId(tuple[0]) + } + if (workflowArgs.mapData) { + tuple[1] = workflowArgs.mapData(tuple[1]) + } + if (workflowArgs.mapPassthrough) { + tuple = tuple.take(2) + workflowArgs.mapPassthrough(tuple.drop(2)) + } + + // check tuple + assert tuple instanceof List : + "Error in module '${key_}': element in channel should be a tuple [id, data, ...otherargs...]\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Expected class: List. Found: tuple.getClass() is ${tuple.getClass()}" + assert tuple.size() >= 2 : + "Error in module '${key_}': expected length of tuple in input channel to be two or greater.\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Found: tuple.size() == ${tuple.size()}" + + // check id field + if (tuple[0] instanceof GString) { + tuple[0] = tuple[0].toString() + } + assert tuple[0] instanceof CharSequence : + "Error in module '${key_}': first element of tuple in channel should be a String\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Found: ${tuple[0]}" + + // match file to input file + if (workflowArgs.auto.simplifyInput && (tuple[1] instanceof Path || tuple[1] instanceof List)) { + def inputFiles = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "input" } + + assert inputFiles.size() == 1 : + "Error in module '${key_}' id '${tuple[0]}'.\n" + + " Anonymous file inputs are only allowed when the process has exactly one file input.\n" + + " Expected: inputFiles.size() == 1. Found: inputFiles.size() is ${inputFiles.size()}" + + tuple[1] = [[ inputFiles[0].plainName, tuple[1] ]].collectEntries() + } + + // check data field + assert tuple[1] instanceof Map : + "Error in module '${key_}' id '${tuple[0]}': second element of tuple in channel should be a Map\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Expected class: Map. Found: tuple[1].getClass() is ${tuple[1].getClass()}" + + // rename keys of data field in tuple + if (workflowArgs.renameKeys) { + assert workflowArgs.renameKeys instanceof Map : + "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + + " Example: renameKeys: ['new_key': 'old_key'].\n" + + " Expected class: Map. Found: renameKeys.getClass() is ${workflowArgs.renameKeys.getClass()}" + assert tuple[1] instanceof Map : + "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + + " Expected class: Map. Found: tuple[1].getClass() is ${tuple[1].getClass()}" + + // TODO: allow renameKeys to be a function? + workflowArgs.renameKeys.each { newKey, oldKey -> + assert newKey instanceof CharSequence : + "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + + " Example: renameKeys: ['new_key': 'old_key'].\n" + + " Expected class of newKey: String. Found: newKey.getClass() is ${newKey.getClass()}" + assert oldKey instanceof CharSequence : + "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + + " Example: renameKeys: ['new_key': 'old_key'].\n" + + " Expected class of oldKey: String. Found: oldKey.getClass() is ${oldKey.getClass()}" + assert tuple[1].containsKey(oldKey) : + "Error renaming data keys in module '${key}' id '${tuple[0]}'.\n" + + " Key '$oldKey' is missing in the data map. tuple[1].keySet() is '${tuple[1].keySet()}'" + tuple[1].put(newKey, tuple[1][oldKey]) + } + tuple[1].keySet().removeAll(workflowArgs.renameKeys.collect{ newKey, oldKey -> oldKey }) + } + tuple + } + + + def chRun = null + def chPassthrough = null + if (workflowArgs.runIf) { + def runIfBranch = chModified.branch{ tup -> + run: workflowArgs.runIf(tup[0], tup[1]) + passthrough: true + } + chRun = runIfBranch.run + chPassthrough = runIfBranch.passthrough + } else { + chRun = chModified + chPassthrough = Channel.empty() + } + + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + + def chArgs = workflowArgs.fromState ? + chRunFiltered | map{ + def new_data = workflowArgs.fromState(it.take(2)) + [it[0], new_data] + } : + chRunFiltered | map {tup -> tup.take(2)} + + // fill in defaults + def chArgsWithDefaults = chArgs + | map { tuple -> + def id_ = tuple[0] + def data_ = tuple[1] + + // TODO: could move fromState to here + + // fetch default params from functionality + def defaultArgs = meta.config.allArguments + .findAll { it.containsKey("default") } + .collectEntries { [ it.plainName, it.default ] } + + // fetch overrides in params + def paramArgs = meta.config.allArguments + .findAll { par -> + def argKey = key_ + "__" + par.plainName + params.containsKey(argKey) + } + .collectEntries { [ it.plainName, params[key_ + "__" + it.plainName] ] } + + // fetch overrides in data + def dataArgs = meta.config.allArguments + .findAll { data_.containsKey(it.plainName) } + .collectEntries { [ it.plainName, data_[it.plainName] ] } + + // combine params + def combinedArgs = defaultArgs + paramArgs + workflowArgs.args + dataArgs + + // remove arguments with explicit null values + combinedArgs + .removeAll{_, val -> val == null || val == "viash_no_value" || val == "force_null"} + + combinedArgs = _processInputValues(combinedArgs, meta.config, id_, key_) + + [id_, combinedArgs] + tuple.drop(2) + } + + // TODO: move some of the _meta.join_id wrangling to the safeJoin() function. + def chInitialOutput = chArgsWithDefaults + | _debug(workflowArgs, "processed") + // run workflow + | innerWorkflowFactory(workflowArgs) + // check output tuple + | map { id_, output_ -> + + // see if output map contains metadata + def meta_ = + output_ instanceof Map && output_.containsKey("_meta") ? + output_["_meta"] : + [:] + def join_id = meta_.join_id ?: id_ + + // remove metadata + output_ = output_.findAll{k, v -> k != "_meta"} + + // check value types + output_ = _processOutputValues(output_, meta.config, id_, key_) + + // simplify output if need be + if (workflowArgs.auto.simplifyOutput && output_.size() == 1) { + output_ = output_.values()[0] + } + + [join_id, id_, output_] + } + // | view{"chInitialOutput: ${it.take(3)}"} + + // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) + // input tuple format: [join_id, id, output, prev_state, ...] + // output tuple format: [join_id, id, new_state, ...] + | map{ tup -> + def new_state = workflowArgs.toState(tup.drop(1).take(3)) + tup.take(2) + [new_state] + tup.drop(4) + } + + if (workflowArgs.auto.publish == "state") { + def chPublish = chNewState + // input tuple format: [join_id, id, new_state, ...] + // output tuple format: [join_id, id, new_state] + | map{ tup -> + tup.take(3) + } + + safeJoin(chPublish, chArgsWithDefaults, key_) + // input tuple format: [join_id, id, new_state, orig_state, ...] + // output tuple format: [id, new_state, orig_state] + | map { tup -> + tup.drop(1).take(3) + } + | publishStatesByConfig(key: key_, config: meta.config) + } + + // remove join_id and meta + chReturn = chNewState + | map { tup -> + // input tuple format: [join_id, id, new_state, ...] + // output tuple format: [id, new_state, ...] + tup.drop(1) + } + | _debug(workflowArgs, "output") + | concat(chPassthrough) + + emit: chReturn + } + + def wf = workflowInstance.cloneWithName(key_) + + // add factory function + wf.metaClass.run = { runArgs -> + workflowFactory(runArgs, workflowArgs, meta) + } + // add config to module for later introspection + wf.metaClass.config = meta.config + + return wf +} + +nextflow.enable.dsl=2 + +// START COMPONENT-SPECIFIC CODE + +// create meta object +meta = [ + "resources_dir": moduleDir.toRealPath().normalize(), + "config": processConfig(readJsonBlob('''{ + "name" : "bedtools_links", + "namespace" : "bedtools", + "version" : "main", + "authors" : [ + { + "name" : "Theodoro Gasperin Terra Camargo", + "roles" : [ + "author", + "maintainer" + ], + "info" : { + "links" : { + "email" : "theodorogtc@gmail.com", + "github" : "tgaspe", + "linkedin" : "theodoro-gasperin-terra-camargo" + }, + "organizations" : [ + { + "name" : "Data Intuitive", + "href" : "https://www.data-intuitive.com", + "role" : "Bioinformatician" + } + ] + } + } + ], + "argument_groups" : [ + { + "name" : "Inputs", + "arguments" : [ + { + "type" : "file", + "name" : "--input", + "alternatives" : [ + "-i" + ], + "description" : "Input file (bed/gff/vcf).", + "must_exist" : true, + "create_parent" : true, + "required" : true, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + } + ] + }, + { + "name" : "Outputs", + "arguments" : [ + { + "type" : "file", + "name" : "--output", + "alternatives" : [ + "-o" + ], + "description" : "Output HTML file to be written.", + "must_exist" : true, + "create_parent" : true, + "required" : false, + "direction" : "output", + "multiple" : false, + "multiple_sep" : ";" + } + ] + }, + { + "name" : "Options", + "description" : "By default, the links created will point to human (hg18) UCSC browser.\nIf you have a local mirror, you can override this behavior by supplying\nthe -base, -org, and -db options.\n\nFor example, if the URL of your local mirror for mouse MM9 is called: \nhttp://mymirror.myuniversity.edu, then you would use the following:\n--base_url http://mymirror.myuniversity.edu\n--organism mouse\n--database mm9\n", + "arguments" : [ + { + "type" : "string", + "name" : "--base_url", + "alternatives" : [ + "-base" + ], + "description" : "The “basename” for the UCSC browser.\n", + "default" : [ + "http://genome.ucsc.edu" + ], + "required" : false, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + }, + { + "type" : "string", + "name" : "--organism", + "alternatives" : [ + "-org" + ], + "description" : "The organism (e.g. mouse, human). \n", + "default" : [ + "human" + ], + "required" : false, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + }, + { + "type" : "string", + "name" : "--database", + "alternatives" : [ + "-db" + ], + "description" : "The genome build. \n", + "default" : [ + "hg18" + ], + "required" : false, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + } + ] + } + ], + "resources" : [ + { + "type" : "bash_script", + "path" : "script.sh", + "is_executable" : true + } + ], + "description" : "Creates an HTML file with links to an instance of the UCSC Genome Browser for all features / intervals in a file. \nThis is useful for cases when one wants to manually inspect through a large set of annotations or features.\n", + "test_resources" : [ + { + "type" : "bash_script", + "path" : "test.sh", + "is_executable" : true + } + ], + "status" : "enabled", + "requirements" : { + "commands" : [ + "ps" + ] + }, + "keywords" : [ + "Links", + "BED", + "GFF", + "VCF" + ], + "license" : "MIT", + "references" : { + "doi" : [ + "10.1093/bioinformatics/btq033" + ] + }, + "links" : { + "repository" : "https://github.com/arq5x/bedtools2", + "homepage" : "https://bedtools.readthedocs.io/en/latest/#", + "documentation" : "https://bedtools.readthedocs.io/en/latest/content/tools/links.html", + "issue_tracker" : "https://github.com/arq5x/bedtools2/issues" + }, + "runners" : [ + { + "type" : "executable", + "id" : "executable", + "docker_setup_strategy" : "ifneedbepullelsecachedbuild" + }, + { + "type" : "nextflow", + "id" : "nextflow", + "directives" : { + "tag" : "$id" + }, + "auto" : { + "simplifyInput" : true, + "simplifyOutput" : false, + "transcript" : false, + "publish" : false + }, + "config" : { + "labels" : { + "mem1gb" : "memory = 1000000000.B", + "mem2gb" : "memory = 2000000000.B", + "mem5gb" : "memory = 5000000000.B", + "mem10gb" : "memory = 10000000000.B", + "mem20gb" : "memory = 20000000000.B", + "mem50gb" : "memory = 50000000000.B", + "mem100gb" : "memory = 100000000000.B", + "mem200gb" : "memory = 200000000000.B", + "mem500gb" : "memory = 500000000000.B", + "mem1tb" : "memory = 1000000000000.B", + "mem2tb" : "memory = 2000000000000.B", + "mem5tb" : "memory = 5000000000000.B", + "mem10tb" : "memory = 10000000000000.B", + "mem20tb" : "memory = 20000000000000.B", + "mem50tb" : "memory = 50000000000000.B", + "mem100tb" : "memory = 100000000000000.B", + "mem200tb" : "memory = 200000000000000.B", + "mem500tb" : "memory = 500000000000000.B", + "mem1gib" : "memory = 1073741824.B", + "mem2gib" : "memory = 2147483648.B", + "mem4gib" : "memory = 4294967296.B", + "mem8gib" : "memory = 8589934592.B", + "mem16gib" : "memory = 17179869184.B", + "mem32gib" : "memory = 34359738368.B", + "mem64gib" : "memory = 68719476736.B", + "mem128gib" : "memory = 137438953472.B", + "mem256gib" : "memory = 274877906944.B", + "mem512gib" : "memory = 549755813888.B", + "mem1tib" : "memory = 1099511627776.B", + "mem2tib" : "memory = 2199023255552.B", + "mem4tib" : "memory = 4398046511104.B", + "mem8tib" : "memory = 8796093022208.B", + "mem16tib" : "memory = 17592186044416.B", + "mem32tib" : "memory = 35184372088832.B", + "mem64tib" : "memory = 70368744177664.B", + "mem128tib" : "memory = 140737488355328.B", + "mem256tib" : "memory = 281474976710656.B", + "mem512tib" : "memory = 562949953421312.B", + "cpu1" : "cpus = 1", + "cpu2" : "cpus = 2", + "cpu5" : "cpus = 5", + "cpu10" : "cpus = 10", + "cpu20" : "cpus = 20", + "cpu50" : "cpus = 50", + "cpu100" : "cpus = 100", + "cpu200" : "cpus = 200", + "cpu500" : "cpus = 500", + "cpu1000" : "cpus = 1000" + } + }, + "debug" : false, + "container" : "docker" + } + ], + "engines" : [ + { + "type" : "docker", + "id" : "docker", + "image" : "debian:stable-slim", + "target_registry" : "images.viash-hub.com", + "target_tag" : "main", + "namespace_separator" : "/", + "setup" : [ + { + "type" : "apt", + "packages" : [ + "bedtools", + "procps" + ], + "interactive" : false + }, + { + "type" : "docker", + "run" : [ + "echo \\"bedtools: \\\\\\"$(bedtools --version | sed -n 's/^bedtools //p')\\\\\\"\\" > /var/software_versions.txt\n" + ] + } + ] + }, + { + "type" : "native", + "id" : "native" + } + ], + "build_info" : { + "config" : "/workdir/root/repo/src/bedtools/bedtools_links/config.vsh.yaml", + "runner" : "nextflow", + "engine" : "docker|native", + "output" : "target/nextflow/bedtools/bedtools_links", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", + "git_remote" : "https://github.com/viash-hub/biobox" + }, + "package_config" : { + "name" : "biobox", + "version" : "main", + "description" : "A collection of bioinformatics tools for working with sequence data.\n", + "viash_version" : "0.9.0-RC7", + "source" : "src", + "target" : "target", + "config_mods" : [ + ".requirements.commands := ['ps']\n", + ".engines += { type: \\"native\\" }", + ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", + ".engines[.type == 'docker'].target_tag := 'main'" + ], + "keywords" : [ + "bioinformatics", + "modules", + "sequencing" + ], + "license" : "MIT", + "organization" : "vsh", + "links" : { + "repository" : "https://github.com/viash-hub/biobox", + "issue_tracker" : "https://github.com/viash-hub/biobox/issues" + } + } +}''')) +] + +// resolve dependencies dependencies (if any) + + +// inner workflow +// inner workflow hook +def innerWorkflowFactory(args) { + def rawScript = '''set -e +tempscript=".viash_script.sh" +cat > "$tempscript" << VIASHMAIN +#!/bin/bash + +## VIASH START +# The following code has been auto-generated by Viash. +$( if [ ! -z ${VIASH_PAR_INPUT+x} ]; then echo "${VIASH_PAR_INPUT}" | sed "s#'#'\\"'\\"'#g;s#.*#par_input='&'#" ; else echo "# par_input="; fi ) +$( if [ ! -z ${VIASH_PAR_OUTPUT+x} ]; then echo "${VIASH_PAR_OUTPUT}" | sed "s#'#'\\"'\\"'#g;s#.*#par_output='&'#" ; else echo "# par_output="; fi ) +$( if [ ! -z ${VIASH_PAR_BASE_URL+x} ]; then echo "${VIASH_PAR_BASE_URL}" | sed "s#'#'\\"'\\"'#g;s#.*#par_base_url='&'#" ; else echo "# par_base_url="; fi ) +$( if [ ! -z ${VIASH_PAR_ORGANISM+x} ]; then echo "${VIASH_PAR_ORGANISM}" | sed "s#'#'\\"'\\"'#g;s#.*#par_organism='&'#" ; else echo "# par_organism="; fi ) +$( if [ ! -z ${VIASH_PAR_DATABASE+x} ]; then echo "${VIASH_PAR_DATABASE}" | sed "s#'#'\\"'\\"'#g;s#.*#par_database='&'#" ; else echo "# par_database="; fi ) +$( if [ ! -z ${VIASH_META_NAME+x} ]; then echo "${VIASH_META_NAME}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_name='&'#" ; else echo "# meta_name="; fi ) +$( if [ ! -z ${VIASH_META_FUNCTIONALITY_NAME+x} ]; then echo "${VIASH_META_FUNCTIONALITY_NAME}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_functionality_name='&'#" ; else echo "# meta_functionality_name="; fi ) +$( if [ ! -z ${VIASH_META_RESOURCES_DIR+x} ]; then echo "${VIASH_META_RESOURCES_DIR}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_resources_dir='&'#" ; else echo "# meta_resources_dir="; fi ) +$( if [ ! -z ${VIASH_META_EXECUTABLE+x} ]; then echo "${VIASH_META_EXECUTABLE}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_executable='&'#" ; else echo "# meta_executable="; fi ) +$( if [ ! -z ${VIASH_META_CONFIG+x} ]; then echo "${VIASH_META_CONFIG}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_config='&'#" ; else echo "# meta_config="; fi ) +$( if [ ! -z ${VIASH_META_TEMP_DIR+x} ]; then echo "${VIASH_META_TEMP_DIR}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_temp_dir='&'#" ; else echo "# meta_temp_dir="; fi ) +$( if [ ! -z ${VIASH_META_CPUS+x} ]; then echo "${VIASH_META_CPUS}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_cpus='&'#" ; else echo "# meta_cpus="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_B+x} ]; then echo "${VIASH_META_MEMORY_B}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_b='&'#" ; else echo "# meta_memory_b="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_KB+x} ]; then echo "${VIASH_META_MEMORY_KB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_kb='&'#" ; else echo "# meta_memory_kb="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_MB+x} ]; then echo "${VIASH_META_MEMORY_MB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_mb='&'#" ; else echo "# meta_memory_mb="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_GB+x} ]; then echo "${VIASH_META_MEMORY_GB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_gb='&'#" ; else echo "# meta_memory_gb="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_TB+x} ]; then echo "${VIASH_META_MEMORY_TB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_tb='&'#" ; else echo "# meta_memory_tb="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_PB+x} ]; then echo "${VIASH_META_MEMORY_PB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_pb='&'#" ; else echo "# meta_memory_pb="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_KIB+x} ]; then echo "${VIASH_META_MEMORY_KIB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_kib='&'#" ; else echo "# meta_memory_kib="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_MIB+x} ]; then echo "${VIASH_META_MEMORY_MIB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_mib='&'#" ; else echo "# meta_memory_mib="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_GIB+x} ]; then echo "${VIASH_META_MEMORY_GIB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_gib='&'#" ; else echo "# meta_memory_gib="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_TIB+x} ]; then echo "${VIASH_META_MEMORY_TIB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_tib='&'#" ; else echo "# meta_memory_tib="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_PIB+x} ]; then echo "${VIASH_META_MEMORY_PIB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_pib='&'#" ; else echo "# meta_memory_pib="; fi ) + +## VIASH END + +set -eo pipefail + +# Execute bedtools links +bedtools links \\\\ + \\${par_base_url:+-base "\\$par_base_url"} \\\\ + \\${par_organism:+-org "\\$par_organism"} \\\\ + \\${par_database:+-db "\\$par_database"} \\\\ + -i "\\$par_input" \\\\ + > "\\$par_output" +VIASHMAIN +bash "$tempscript" +''' + + return vdsl3WorkflowFactory(args, meta, rawScript) +} + + + +/** + * Generate a workflow for VDSL3 modules. + * + * This function is called by the workflowFactory() function. + * + * Input channel: [id, input_map] + * Output channel: [id, output_map] + * + * Internally, this workflow will convert the input channel + * to a format which the Nextflow module will be able to handle. + */ +def vdsl3WorkflowFactory(Map args, Map meta, String rawScript) { + def key = args["key"] + def processObj = null + + workflow processWf { + take: input_ + main: + + if (processObj == null) { + processObj = _vdsl3ProcessFactory(args, meta, rawScript) + } + + output_ = input_ + | map { tuple -> + def id = tuple[0] + def data_ = tuple[1] + + if (workflow.stubRun) { + // add id if missing + data_ = [id: 'stub'] + data_ + } + + // process input files separately + def inputPaths = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "input" } + .collect { par -> + def val = data_.containsKey(par.plainName) ? data_[par.plainName] : [] + def inputFiles = [] + if (val == null) { + inputFiles = [] + } else if (val instanceof List) { + inputFiles = val + } else if (val instanceof Path) { + inputFiles = [ val ] + } else { + inputFiles = [] + } + if (!workflow.stubRun) { + // throw error when an input file doesn't exist + inputFiles.each{ file -> + assert file.exists() : + "Error in module '${key}' id '${id}' argument '${par.plainName}'.\n" + + " Required input file does not exist.\n" + + " Path: '$file'.\n" + + " Expected input file to exist" + } + } + inputFiles + } + + // remove input files + def argsExclInputFiles = meta.config.allArguments + .findAll { (it.type != "file" || it.direction != "input") && data_.containsKey(it.plainName) } + .collectEntries { par -> + def parName = par.plainName + def val = data_[parName] + if (par.multiple && val instanceof Collection) { + val = val.join(par.multiple_sep) + } + if (par.direction == "output" && par.type == "file") { + val = val.replaceAll('\\$id', id).replaceAll('\\$key', key) + } + [parName, val] + } + + [ id ] + inputPaths + [ argsExclInputFiles, meta.resources_dir ] + } + | processObj + | map { output -> + def outputFiles = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "output" } + .indexed() + .collectEntries{ index, par -> + def out = output[index + 1] + // strip dummy '.exitcode' file from output (see nextflow-io/nextflow#2678) + if (!out instanceof List || out.size() <= 1) { + if (par.multiple) { + out = [] + } else { + assert !par.required : + "Error in module '${key}' id '${output[0]}' argument '${par.plainName}'.\n" + + " Required output file is missing" + out = null + } + } else if (out.size() == 2 && !par.multiple) { + out = out[1] + } else { + out = out.drop(1) + } + [ par.plainName, out ] + } + + // drop null outputs + outputFiles.removeAll{it.value == null} + + [ output[0], outputFiles ] + } + emit: output_ + } + + return processWf +} + +// depends on: session? +def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) { + // autodetect process key + def wfKey = workflowArgs["key"] + def procKeyPrefix = "${wfKey}_process" + def scriptMeta = nextflow.script.ScriptMeta.current() + def existing = scriptMeta.getProcessNames().findAll{it.startsWith(procKeyPrefix)} + def numbers = existing.collect{it.replace(procKeyPrefix, "0").toInteger()} + def newNumber = (numbers + [-1]).max() + 1 + + def procKey = newNumber == 0 ? procKeyPrefix : "$procKeyPrefix$newNumber" + + if (newNumber > 0) { + log.warn "Key for module '${wfKey}' is duplicated.\n", + "If you run a component multiple times in the same workflow,\n" + + "it's recommended you set a unique key for every call,\n" + + "for example: ${wfKey}.run(key: \"foo\")." + } + + // subset directives and convert to list of tuples + def drctv = workflowArgs.directives + + // TODO: unit test the two commands below + // convert publish array into tags + def valueToStr = { val -> + // ignore closures + if (val instanceof CharSequence) { + if (!val.matches('^[{].*[}]$')) { + '"' + val + '"' + } else { + val + } + } else if (val instanceof List) { + "[" + val.collect{valueToStr(it)}.join(", ") + "]" + } else if (val instanceof Map) { + "[" + val.collect{k, v -> k + ": " + valueToStr(v)}.join(", ") + "]" + } else { + val.inspect() + } + } + + // multiple entries allowed: label, publishdir + def drctvStrs = drctv.collect { key, value -> + if (key in ["label", "publishDir"]) { + value.collect{ val -> + if (val instanceof Map) { + "\n$key " + val.collect{ k, v -> k + ": " + valueToStr(v) }.join(", ") + } else if (val == null) { + "" + } else { + "\n$key " + valueToStr(val) + } + }.join() + } else if (value instanceof Map) { + "\n$key " + value.collect{ k, v -> k + ": " + valueToStr(v) }.join(", ") + } else { + "\n$key " + valueToStr(value) + } + }.join() + + def inputPaths = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "input" } + .collect { ', path(viash_par_' + it.plainName + ', stageAs: "_viash_par/' + it.plainName + '_?/*")' } + .join() + + def outputPaths = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "output" } + .collect { par -> + // insert dummy into every output (see nextflow-io/nextflow#2678) + if (!par.multiple) { + ', path{[".exitcode", args.' + par.plainName + ']}' + } else { + ', path{[".exitcode"] + args.' + par.plainName + '}' + } + } + .join() + + // TODO: move this functionality somewhere else? + if (workflowArgs.auto.transcript) { + outputPaths = outputPaths + ', path{[".exitcode", ".command*"]}' + } else { + outputPaths = outputPaths + ', path{[".exitcode"]}' + } + + // create dirs for output files (based on BashWrapper.createParentFiles) + def createParentStr = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "output" && it.create_parent } + .collect { par -> + "\${ args.containsKey(\"${par.plainName}\") ? \"mkdir_parent \\\"\" + (args[\"${par.plainName}\"] instanceof String ? args[\"${par.plainName}\"] : args[\"${par.plainName}\"].join('\" \"')) + \"\\\"\" : \"\" }" + } + .join("\n") + + // construct inputFileExports + def inputFileExports = meta.config.allArguments + .findAll { it.type == "file" && it.direction.toLowerCase() == "input" } + .collect { par -> + def viash_par_contents = "(viash_par_${par.plainName} instanceof List ? viash_par_${par.plainName}.join(\"${par.multiple_sep}\") : viash_par_${par.plainName})" + "\n\${viash_par_${par.plainName}.empty ? \"\" : \"export VIASH_PAR_${par.plainName.toUpperCase()}=\\\"\" + ${viash_par_contents} + \"\\\"\"}" + } + + // NOTE: if using docker, use /tmp instead of tmpDir! + def tmpDir = java.nio.file.Paths.get( + System.getenv('NXF_TEMP') ?: + System.getenv('VIASH_TEMP') ?: + System.getenv('VIASH_TMPDIR') ?: + System.getenv('VIASH_TEMPDIR') ?: + System.getenv('VIASH_TMP') ?: + System.getenv('TEMP') ?: + System.getenv('TMPDIR') ?: + System.getenv('TEMPDIR') ?: + System.getenv('TMP') ?: + '/tmp' + ).toAbsolutePath() + + // construct stub + def stub = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "output" } + .collect { par -> + "\${ args.containsKey(\"${par.plainName}\") ? \"touch2 \\\"\" + (args[\"${par.plainName}\"] instanceof String ? args[\"${par.plainName}\"].replace(\"_*\", \"_0\") : args[\"${par.plainName}\"].join('\" \"')) + \"\\\"\" : \"\" }" + } + .join("\n") + + // escape script + def escapedScript = rawScript.replace('\\', '\\\\').replace('$', '\\$').replace('"""', '\\"\\"\\"') + + // publishdir assert + def assertStr = (workflowArgs.auto.publish == true) || workflowArgs.auto.transcript ? + """\nassert task.publishDir.size() > 0: "if auto.publish is true, params.publish_dir needs to be defined.\\n Example: --publish_dir './output/'" """ : + "" + + // generate process string + def procStr = + """nextflow.enable.dsl=2 + | + |process $procKey {$drctvStrs + |input: + | tuple val(id)$inputPaths, val(args), path(resourcesDir, stageAs: ".viash_meta_resources") + |output: + | tuple val("\$id")$outputPaths, optional: true + |stub: + |\"\"\" + |touch2() { mkdir -p "\\\$(dirname "\\\$1")" && touch "\\\$1" ; } + |$stub + |\"\"\" + |script:$assertStr + |def escapeText = { s -> s.toString().replaceAll('([`"])', '\\\\\\\\\$1') } + |def parInject = args + | .findAll{key, value -> value != null} + | .collect{key, value -> "export VIASH_PAR_\${key.toUpperCase()}=\\\"\${escapeText(value)}\\\""} + | .join("\\n") + |\"\"\" + |# meta exports + |export VIASH_META_RESOURCES_DIR="\${resourcesDir}" + |export VIASH_META_TEMP_DIR="${['docker', 'podman', 'charliecloud'].any{ it == workflow.containerEngine } ? '/tmp' : tmpDir}" + |export VIASH_META_NAME="${meta.config.name}" + |# export VIASH_META_EXECUTABLE="\\\$VIASH_META_RESOURCES_DIR/\\\$VIASH_META_NAME" + |export VIASH_META_CONFIG="\\\$VIASH_META_RESOURCES_DIR/.config.vsh.yaml" + |\${task.cpus ? "export VIASH_META_CPUS=\$task.cpus" : "" } + |\${task.memory?.bytes != null ? "export VIASH_META_MEMORY_B=\$task.memory.bytes" : "" } + |if [ ! -z \\\${VIASH_META_MEMORY_B+x} ]; then + | export VIASH_META_MEMORY_KB=\\\$(( (\\\$VIASH_META_MEMORY_B+999) / 1000 )) + | export VIASH_META_MEMORY_MB=\\\$(( (\\\$VIASH_META_MEMORY_KB+999) / 1000 )) + | export VIASH_META_MEMORY_GB=\\\$(( (\\\$VIASH_META_MEMORY_MB+999) / 1000 )) + | export VIASH_META_MEMORY_TB=\\\$(( (\\\$VIASH_META_MEMORY_GB+999) / 1000 )) + | export VIASH_META_MEMORY_PB=\\\$(( (\\\$VIASH_META_MEMORY_TB+999) / 1000 )) + | export VIASH_META_MEMORY_KIB=\\\$(( (\\\$VIASH_META_MEMORY_B+1023) / 1024 )) + | export VIASH_META_MEMORY_MIB=\\\$(( (\\\$VIASH_META_MEMORY_KIB+1023) / 1024 )) + | export VIASH_META_MEMORY_GIB=\\\$(( (\\\$VIASH_META_MEMORY_MIB+1023) / 1024 )) + | export VIASH_META_MEMORY_TIB=\\\$(( (\\\$VIASH_META_MEMORY_GIB+1023) / 1024 )) + | export VIASH_META_MEMORY_PIB=\\\$(( (\\\$VIASH_META_MEMORY_TIB+1023) / 1024 )) + |fi + | + |# meta synonyms + |export VIASH_TEMP="\\\$VIASH_META_TEMP_DIR" + |export TEMP_DIR="\\\$VIASH_META_TEMP_DIR" + | + |# create output dirs if need be + |function mkdir_parent { + | for file in "\\\$@"; do + | mkdir -p "\\\$(dirname "\\\$file")" + | done + |} + |$createParentStr + | + |# argument exports${inputFileExports.join()} + |\$parInject + | + |# process script + |${escapedScript} + |\"\"\" + |} + |""".stripMargin() + + // TODO: print on debug + // if (workflowArgs.debug == true) { + // println("######################\n$procStr\n######################") + // } + + // write process to temp file + def tempFile = java.nio.file.Files.createTempFile("viash-process-${procKey}-", ".nf") + addShutdownHook { java.nio.file.Files.deleteIfExists(tempFile) } + tempFile.text = procStr + + // create process from temp file + def binding = new nextflow.script.ScriptBinding([:]) + def session = nextflow.Nextflow.getSession() + def parser = new nextflow.script.ScriptParser(session) + .setModule(true) + .setBinding(binding) + def moduleScript = parser.runScript(tempFile) + .getScript() + + // register module in meta + def module = new nextflow.script.IncludeDef.Module(name: procKey) + scriptMeta.addModule(moduleScript, module.name, module.alias) + + // retrieve and return process from meta + return scriptMeta.getProcess(procKey) +} + +// defaults +meta["defaults"] = [ + // key to be used to trace the process and determine output names + key: null, + + // fixed arguments to be passed to script + args: [:], + + // default directives + directives: readJsonBlob('''{ + "container" : { + "registry" : "images.viash-hub.com", + "image" : "vsh/biobox/bedtools/bedtools_links", + "tag" : "main" + }, + "tag" : "$id" +}'''), + + // auto settings + auto: readJsonBlob('''{ + "simplifyInput" : true, + "simplifyOutput" : false, + "transcript" : false, + "publish" : false +}'''), + + // Apply a map over the incoming tuple + // Example: `{ tup -> [ tup[0], [input: tup[1].output] ] + tup.drop(2) }` + map: null, + + // Apply a map over the ID element of a tuple (i.e. the first element) + // Example: `{ id -> id + "_foo" }` + mapId: null, + + // Apply a map over the data element of a tuple (i.e. the second element) + // Example: `{ data -> [ input: data.output ] }` + mapData: null, + + // Apply a map over the passthrough elements of a tuple (i.e. the tuple excl. the first two elements) + // Example: `{ pt -> pt.drop(1) }` + mapPassthrough: null, + + // Filter the channel + // Example: `{ tup -> tup[0] == "foo" }` + filter: null, + + // Choose whether or not to run the component on the tuple if the condition is true. + // Otherwise, the tuple will be passed through. + // Example: `{ tup -> tup[0] != "skip_this" }` + runIf: null, + + // Rename keys in the data field of the tuple (i.e. the second element) + // Will likely be deprecated in favour of `fromState`. + // Example: `[ "new_key": "old_key" ]` + renameKeys: null, + + // Fetch data from the state and pass it to the module without altering the current state. + // + // `fromState` should be `null`, `List[String]`, `Map[String, String]` or a function. + // + // - If it is `null`, the state will be passed to the module as is. + // - If it is a `List[String]`, the data will be the values of the state at the given keys. + // - If it is a `Map[String, String]`, the data will be the values of the state at the given keys, with the keys renamed according to the map. + // - If it is a function, the tuple (`[id, state]`) in the channel will be passed to the function, and the result will be used as the data. + // + // Example: `{ id, state -> [input: state.fastq_file] }` + // Default: `null` + fromState: null, + + // Determine how the state should be updated after the module has been run. + // + // `toState` should be `null`, `List[String]`, `Map[String, String]` or a function. + // + // - If it is `null`, the state will be replaced with the output of the module. + // - If it is a `List[String]`, the state will be updated with the values of the data at the given keys. + // - If it is a `Map[String, String]`, the state will be updated with the values of the data at the given keys, with the keys renamed according to the map. + // - If it is a function, a tuple (`[id, output, state]`) will be passed to the function, and the result will be used as the new state. + // + // Example: `{ id, output, state -> state + [counts: state.output] }` + // Default: `{ id, output, state -> output }` + toState: null, + + // Whether or not to print debug messages + // Default: `false` + debug: false +] + +// initialise default workflow +meta["workflow"] = workflowFactory([key: meta.config.name], meta.defaults, meta) + +// add workflow to environment +nextflow.script.ScriptMeta.current().addDefinition(meta.workflow) + +// anonymous workflow for running this module as a standalone +workflow { + // add id argument if it's not already in the config + // TODO: deep copy + def newConfig = deepClone(meta.config) + def newParams = deepClone(params) + + def argsContainsId = newConfig.allArguments.any{it.plainName == "id"} + if (!argsContainsId) { + def idArg = [ + 'name': '--id', + 'required': false, + 'type': 'string', + 'description': 'A unique id for every entry.', + 'multiple': false + ] + newConfig.arguments.add(0, idArg) + newConfig = processConfig(newConfig) + } + if (!newParams.containsKey("id")) { + newParams.id = "run" + } + + helpMessage(newConfig) + + channelFromParams(newParams, newConfig) + // make sure id is not in the state if id is not in the args + | map {id, state -> + if (!argsContainsId) { + [id, state.findAll{k, v -> k != "id"}] + } else { + [id, state] + } + } + | meta.workflow.run( + auto: [ publish: "state" ] + ) +} + +// END COMPONENT-SPECIFIC CODE diff --git a/target/nextflow/bedtools/bedtools_links/nextflow.config b/target/nextflow/bedtools/bedtools_links/nextflow.config new file mode 100644 index 00000000..250d0705 --- /dev/null +++ b/target/nextflow/bedtools/bedtools_links/nextflow.config @@ -0,0 +1,126 @@ +manifest { + name = 'bedtools/bedtools_links' + mainScript = 'main.nf' + nextflowVersion = '!>=20.12.1-edge' + version = 'main' + description = 'Creates an HTML file with links to an instance of the UCSC Genome Browser for all features / intervals in a file. \nThis is useful for cases when one wants to manually inspect through a large set of annotations or features.\n' + author = 'Theodoro Gasperin Terra Camargo' +} + +process.container = 'nextflow/bash:latest' + +// detect tempdir +tempDir = java.nio.file.Paths.get( + System.getenv('NXF_TEMP') ?: + System.getenv('VIASH_TEMP') ?: + System.getenv('TEMPDIR') ?: + System.getenv('TMPDIR') ?: + '/tmp' +).toAbsolutePath() + +profiles { + no_publish { + process { + withName: '.*' { + publishDir = [ + enabled: false + ] + } + } + } + mount_temp { + docker.temp = tempDir + podman.temp = tempDir + charliecloud.temp = tempDir + } + docker { + docker.enabled = true + // docker.userEmulation = true + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + singularity { + singularity.enabled = true + singularity.autoMounts = true + docker.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + podman { + podman.enabled = true + docker.enabled = false + singularity.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + shifter { + shifter.enabled = true + docker.enabled = false + singularity.enabled = false + podman.enabled = false + charliecloud.enabled = false + } + charliecloud { + charliecloud.enabled = true + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + } +} + +process{ + withLabel: mem1gb { memory = 1000000000.B } + withLabel: mem2gb { memory = 2000000000.B } + withLabel: mem5gb { memory = 5000000000.B } + withLabel: mem10gb { memory = 10000000000.B } + withLabel: mem20gb { memory = 20000000000.B } + withLabel: mem50gb { memory = 50000000000.B } + withLabel: mem100gb { memory = 100000000000.B } + withLabel: mem200gb { memory = 200000000000.B } + withLabel: mem500gb { memory = 500000000000.B } + withLabel: mem1tb { memory = 1000000000000.B } + withLabel: mem2tb { memory = 2000000000000.B } + withLabel: mem5tb { memory = 5000000000000.B } + withLabel: mem10tb { memory = 10000000000000.B } + withLabel: mem20tb { memory = 20000000000000.B } + withLabel: mem50tb { memory = 50000000000000.B } + withLabel: mem100tb { memory = 100000000000000.B } + withLabel: mem200tb { memory = 200000000000000.B } + withLabel: mem500tb { memory = 500000000000000.B } + withLabel: mem1gib { memory = 1073741824.B } + withLabel: mem2gib { memory = 2147483648.B } + withLabel: mem4gib { memory = 4294967296.B } + withLabel: mem8gib { memory = 8589934592.B } + withLabel: mem16gib { memory = 17179869184.B } + withLabel: mem32gib { memory = 34359738368.B } + withLabel: mem64gib { memory = 68719476736.B } + withLabel: mem128gib { memory = 137438953472.B } + withLabel: mem256gib { memory = 274877906944.B } + withLabel: mem512gib { memory = 549755813888.B } + withLabel: mem1tib { memory = 1099511627776.B } + withLabel: mem2tib { memory = 2199023255552.B } + withLabel: mem4tib { memory = 4398046511104.B } + withLabel: mem8tib { memory = 8796093022208.B } + withLabel: mem16tib { memory = 17592186044416.B } + withLabel: mem32tib { memory = 35184372088832.B } + withLabel: mem64tib { memory = 70368744177664.B } + withLabel: mem128tib { memory = 140737488355328.B } + withLabel: mem256tib { memory = 281474976710656.B } + withLabel: mem512tib { memory = 562949953421312.B } + withLabel: cpu1 { cpus = 1 } + withLabel: cpu2 { cpus = 2 } + withLabel: cpu5 { cpus = 5 } + withLabel: cpu10 { cpus = 10 } + withLabel: cpu20 { cpus = 20 } + withLabel: cpu50 { cpus = 50 } + withLabel: cpu100 { cpus = 100 } + withLabel: cpu200 { cpus = 200 } + withLabel: cpu500 { cpus = 500 } + withLabel: cpu1000 { cpus = 1000 } +} + + diff --git a/target/nextflow/bedtools/bedtools_links/nextflow_schema.json b/target/nextflow/bedtools/bedtools_links/nextflow_schema.json new file mode 100644 index 00000000..5a468d62 --- /dev/null +++ b/target/nextflow/bedtools/bedtools_links/nextflow_schema.json @@ -0,0 +1,142 @@ +{ +"$schema": "http://json-schema.org/draft-07/schema", +"title": "bedtools_links", +"description": "Creates an HTML file with links to an instance of the UCSC Genome Browser for all features / intervals in a file. \nThis is useful for cases when one wants to manually inspect through a large set of annotations or features.\n", +"type": "object", +"definitions": { + + + + "inputs" : { + "title": "Inputs", + "type": "object", + "description": "No description", + "properties": { + + + "input": { + "type": + "string", + "description": "Type: `file`, required. Input file (bed/gff/vcf)", + "help_text": "Type: `file`, required. Input file (bed/gff/vcf)." + + } + + +} +}, + + + "outputs" : { + "title": "Outputs", + "type": "object", + "description": "No description", + "properties": { + + + "output": { + "type": + "string", + "description": "Type: `file`, default: `$id.$key.output.output`. Output HTML file to be written", + "help_text": "Type: `file`, default: `$id.$key.output.output`. Output HTML file to be written." + , + "default": "$id.$key.output.output" + } + + +} +}, + + + "options" : { + "title": "Options", + "type": "object", + "description": "By default, the links created will point to human (hg18) UCSC browser.\nIf you have a local mirror, you can override this behavior by supplying\nthe -base, -org, and -db options.\n\nFor example, if the URL of your local mirror for mouse MM9 is called: \nhttp://mymirror.myuniversity.edu, then you would use the following:\n--base_url http://mymirror.myuniversity.edu\n--organism mouse\n--database mm9\n", + "properties": { + + + "base_url": { + "type": + "string", + "description": "Type: `string`, default: `http://genome.ucsc.edu`. The \u201cbasename\u201d for the UCSC browser", + "help_text": "Type: `string`, default: `http://genome.ucsc.edu`. The \u201cbasename\u201d for the UCSC browser.\n" + , + "default": "http://genome.ucsc.edu" + } + + + , + "organism": { + "type": + "string", + "description": "Type: `string`, default: `human`. The organism (e", + "help_text": "Type: `string`, default: `human`. The organism (e.g. mouse, human). \n" + , + "default": "human" + } + + + , + "database": { + "type": + "string", + "description": "Type: `string`, default: `hg18`. The genome build", + "help_text": "Type: `string`, default: `hg18`. The genome build. \n" + , + "default": "hg18" + } + + +} +}, + + + "nextflow input-output arguments" : { + "title": "Nextflow input-output arguments", + "type": "object", + "description": "Input/output parameters for Nextflow itself. Please note that both publishDir and publish_dir are supported but at least one has to be configured.", + "properties": { + + + "publish_dir": { + "type": + "string", + "description": "Type: `string`, required, example: `output/`. Path to an output directory", + "help_text": "Type: `string`, required, example: `output/`. Path to an output directory." + + } + + + , + "param_list": { + "type": + "string", + "description": "Type: `string`, example: `my_params.yaml`. Allows inputting multiple parameter sets to initialise a Nextflow channel", + "help_text": "Type: `string`, example: `my_params.yaml`. Allows inputting multiple parameter sets to initialise a Nextflow channel. A `param_list` can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob.\n\n* A list of maps (as-is) where the keys of each map corresponds to the arguments of the pipeline. Example: in a `nextflow.config` file: `param_list: [ [\u0027id\u0027: \u0027foo\u0027, \u0027input\u0027: \u0027foo.txt\u0027], [\u0027id\u0027: \u0027bar\u0027, \u0027input\u0027: \u0027bar.txt\u0027] ]`.\n* A csv file should have column names which correspond to the different arguments of this pipeline. Example: `--param_list data.csv` with columns `id,input`.\n* A json or a yaml file should be a list of maps, each of which has keys corresponding to the arguments of the pipeline. Example: `--param_list data.json` with contents `[ {\u0027id\u0027: \u0027foo\u0027, \u0027input\u0027: \u0027foo.txt\u0027}, {\u0027id\u0027: \u0027bar\u0027, \u0027input\u0027: \u0027bar.txt\u0027} ]`.\n* A yaml blob can also be passed directly as a string. Example: `--param_list \"[ {\u0027id\u0027: \u0027foo\u0027, \u0027input\u0027: \u0027foo.txt\u0027}, {\u0027id\u0027: \u0027bar\u0027, \u0027input\u0027: \u0027bar.txt\u0027} ]\"`.\n\nWhen passing a csv, json or yaml file, relative path names are relativized to the location of the parameter file. No relativation is performed when `param_list` is a list of maps (as-is) or a yaml blob.", + "hidden": true + + } + + +} +} +}, +"allOf": [ + + { + "$ref": "#/definitions/inputs" + }, + + { + "$ref": "#/definitions/outputs" + }, + + { + "$ref": "#/definitions/options" + }, + + { + "$ref": "#/definitions/nextflow input-output arguments" + } +] +} diff --git a/target/nextflow/bedtools/bedtools_merge/.config.vsh.yaml b/target/nextflow/bedtools/bedtools_merge/.config.vsh.yaml new file mode 100644 index 00000000..ef9a3f17 --- /dev/null +++ b/target/nextflow/bedtools/bedtools_merge/.config.vsh.yaml @@ -0,0 +1,305 @@ +name: "bedtools_merge" +namespace: "bedtools" +version: "main" +authors: +- name: "Theodoro Gasperin Terra Camargo" + roles: + - "author" + - "maintainer" + info: + links: + email: "theodorogtc@gmail.com" + github: "tgaspe" + linkedin: "theodoro-gasperin-terra-camargo" + organizations: + - name: "Data Intuitive" + href: "https://www.data-intuitive.com" + role: "Bioinformatician" +argument_groups: +- name: "Inputs" + arguments: + - type: "file" + name: "--input" + alternatives: + - "-i" + description: "Input file (BED/GFF/VCF) to be merged." + info: null + must_exist: true + create_parent: true + required: true + direction: "input" + multiple: false + multiple_sep: ";" +- name: "Outputs" + arguments: + - type: "file" + name: "--output" + description: "Output merged file BED to be written." + info: null + must_exist: true + create_parent: true + required: true + direction: "output" + multiple: false + multiple_sep: ";" +- name: "Options" + arguments: + - type: "boolean_true" + name: "--strand" + alternatives: + - "-s" + description: "Force strandedness. That is, only merge features\nthat are on the\ + \ same strand.\n- By default, merging is done without respect to strand.\n" + info: null + direction: "input" + - type: "string" + name: "--specific_strand" + alternatives: + - "-S" + description: "Force merge for one specific strand only.\nFollow with + or - to\ + \ force merge from only\nthe forward or reverse strand, respectively.\n- By\ + \ default, merging is done without respect to strand.\n" + info: null + required: false + choices: + - "+" + - "-" + direction: "input" + multiple: false + multiple_sep: ";" + - type: "integer" + name: "--distance" + alternatives: + - "-d" + description: "Maximum distance between features allowed for features\nto be merged.\n\ + - Def. 0. That is, overlapping & book-ended features are merged.\n- (INTEGER)\n\ + - Note: negative values enforce the number of b.p. required for overlap.\n" + info: null + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "integer" + name: "--columns" + alternatives: + - "-c" + description: "Specify columns from the B file to map onto intervals in A.\nDefault:\ + \ 5.\nMultiple columns can be specified in a comma-delimited list.\n" + info: null + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "string" + name: "--operation" + alternatives: + - "-o" + description: "Specify the operation that should be applied to -c.\nValid operations:\n\ + \ sum, min, max, absmin, absmax,\n mean, median, mode, antimode\n stdev,\ + \ sstdev\n collapse (i.e., print a delimited list (duplicates allowed)),\ + \ \n distinct (i.e., print a delimited list (NO duplicates allowed)), \n\ + \ distinct_sort_num (as distinct, sorted numerically, ascending),\n distinct_sort_num_desc\ + \ (as distinct, sorted numerically, desscending),\n distinct_only (delimited\ + \ list of only unique values),\n count\n count_distinct (i.e., a count\ + \ of the unique values in the column), \n first (i.e., just the first value\ + \ in the column), \n last (i.e., just the last value in the column), \nDefault:\ + \ sum\nMultiple operations can be specified in a comma-delimited list.\n\nIf\ + \ there is only column, but multiple operations, all operations will be\napplied\ + \ on that column. Likewise, if there is only one operation, but\nmultiple columns,\ + \ that operation will be applied to all columns.\nOtherwise, the number of columns\ + \ must match the the number of operations,\nand will be applied in respective\ + \ order.\nE.g., \"-c 5,4,6 -o sum,mean,count\" will give the sum of column 5,\n\ + the mean of column 4, and the count of column 6.\nThe order of output columns\ + \ will match the ordering given in the command.\n" + info: null + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "string" + name: "--delimiter" + alternatives: + - "-delim" + description: "Specify a custom delimiter for the collapse operations.\n" + info: null + example: + - "|" + default: + - "," + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "integer" + name: "--precision" + alternatives: + - "-prec" + description: "Sets the decimal precision for output (Default: 5).\n" + info: null + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "boolean_true" + name: "--bed" + description: "If using BAM input, write output as BED.\n" + info: null + direction: "input" + - type: "boolean_true" + name: "--header" + description: "Print the header from the A file prior to results.\n" + info: null + direction: "input" + - type: "boolean_true" + name: "--no_buffer" + alternatives: + - "-nobuf" + description: "Disable buffered output. Using this option will cause each line\n\ + of output to be printed as it is generated, rather than saved\nin a buffer.\ + \ This will make printing large output files \nnoticeably slower, but can be\ + \ useful in conjunction with\nother software tools and scripts that need to\ + \ process one\nline of bedtools output at a time.\n" + info: null + direction: "input" +resources: +- type: "bash_script" + path: "script.sh" + is_executable: true +description: "Merges overlapping BED/GFF/VCF entries into a single interval.\n" +test_resources: +- type: "bash_script" + path: "test.sh" + is_executable: true +- type: "file" + path: "test_data" +info: null +status: "enabled" +requirements: + commands: + - "ps" +license: "MIT" +references: + doi: + - "10.1093/bioinformatics/btq033" +links: + repository: "https://github.com/arq5x/bedtools2" + homepage: "https://bedtools.readthedocs.io/en/latest/#" + documentation: "https://bedtools.readthedocs.io/en/latest/content/tools/merge.html" + issue_tracker: "https://github.com/arq5x/bedtools2/issues" +runners: +- type: "executable" + id: "executable" + docker_setup_strategy: "ifneedbepullelsecachedbuild" +- type: "nextflow" + id: "nextflow" + directives: + tag: "$id" + auto: + simplifyInput: true + simplifyOutput: false + transcript: false + publish: false + config: + labels: + mem1gb: "memory = 1000000000.B" + mem2gb: "memory = 2000000000.B" + mem5gb: "memory = 5000000000.B" + mem10gb: "memory = 10000000000.B" + mem20gb: "memory = 20000000000.B" + mem50gb: "memory = 50000000000.B" + mem100gb: "memory = 100000000000.B" + mem200gb: "memory = 200000000000.B" + mem500gb: "memory = 500000000000.B" + mem1tb: "memory = 1000000000000.B" + mem2tb: "memory = 2000000000000.B" + mem5tb: "memory = 5000000000000.B" + mem10tb: "memory = 10000000000000.B" + mem20tb: "memory = 20000000000000.B" + mem50tb: "memory = 50000000000000.B" + mem100tb: "memory = 100000000000000.B" + mem200tb: "memory = 200000000000000.B" + mem500tb: "memory = 500000000000000.B" + mem1gib: "memory = 1073741824.B" + mem2gib: "memory = 2147483648.B" + mem4gib: "memory = 4294967296.B" + mem8gib: "memory = 8589934592.B" + mem16gib: "memory = 17179869184.B" + mem32gib: "memory = 34359738368.B" + mem64gib: "memory = 68719476736.B" + mem128gib: "memory = 137438953472.B" + mem256gib: "memory = 274877906944.B" + mem512gib: "memory = 549755813888.B" + mem1tib: "memory = 1099511627776.B" + mem2tib: "memory = 2199023255552.B" + mem4tib: "memory = 4398046511104.B" + mem8tib: "memory = 8796093022208.B" + mem16tib: "memory = 17592186044416.B" + mem32tib: "memory = 35184372088832.B" + mem64tib: "memory = 70368744177664.B" + mem128tib: "memory = 140737488355328.B" + mem256tib: "memory = 281474976710656.B" + mem512tib: "memory = 562949953421312.B" + cpu1: "cpus = 1" + cpu2: "cpus = 2" + cpu5: "cpus = 5" + cpu10: "cpus = 10" + cpu20: "cpus = 20" + cpu50: "cpus = 50" + cpu100: "cpus = 100" + cpu200: "cpus = 200" + cpu500: "cpus = 500" + cpu1000: "cpus = 1000" + debug: false + container: "docker" +engines: +- type: "docker" + id: "docker" + image: "debian:stable-slim" + target_registry: "images.viash-hub.com" + target_tag: "main" + namespace_separator: "/" + setup: + - type: "apt" + packages: + - "bedtools" + - "procps" + interactive: false + - type: "docker" + run: + - "echo \"bedtools: \\\"$(bedtools --version | sed -n 's/^bedtools //p')\\\"\"\ + \ > /var/software_versions.txt\n" + entrypoint: [] + cmd: null +- type: "native" + id: "native" +build_info: + config: "src/bedtools/bedtools_merge/config.vsh.yaml" + runner: "nextflow" + engine: "docker|native" + output: "target/nextflow/bedtools/bedtools_merge" + executable: "target/nextflow/bedtools/bedtools_merge/main.nf" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" + git_remote: "https://github.com/viash-hub/biobox" +package_config: + name: "biobox" + version: "main" + description: "A collection of bioinformatics tools for working with sequence data.\n" + info: null + viash_version: "0.9.0-RC7" + source: "src" + target: "target" + config_mods: + - ".requirements.commands := ['ps']\n" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'main'" + keywords: + - "bioinformatics" + - "modules" + - "sequencing" + license: "MIT" + organization: "vsh" + links: + repository: "https://github.com/viash-hub/biobox" + issue_tracker: "https://github.com/viash-hub/biobox/issues" diff --git a/target/nextflow/bedtools/bedtools_merge/main.nf b/target/nextflow/bedtools/bedtools_merge/main.nf new file mode 100644 index 00000000..4198581d --- /dev/null +++ b/target/nextflow/bedtools/bedtools_merge/main.nf @@ -0,0 +1,3713 @@ +// bedtools_merge main +// +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a +// derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from +// Data Intuitive. +// +// The component may contain files which fall under a different license. The +// authors of this component should specify the license in the header of such +// files, or include a separate license file detailing the licenses of all included +// files. +// +// Component authors: +// * Theodoro Gasperin Terra Camargo (author, maintainer) + +//////////////////////////// +// VDSL3 helper functions // +//////////////////////////// + +// helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_checkArgumentType.nf' +class UnexpectedArgumentTypeException extends Exception { + String errorIdentifier + String stage + String plainName + String expectedClass + String foundClass + + // ${key ? " in module '$key'" : ""}${id ? " id '$id'" : ""} + UnexpectedArgumentTypeException(String errorIdentifier, String stage, String plainName, String expectedClass, String foundClass) { + super("Error${errorIdentifier ? " $errorIdentifier" : ""}:${stage ? " $stage" : "" } argument '${plainName}' has the wrong type. " + + "Expected type: ${expectedClass}. Found type: ${foundClass}") + this.errorIdentifier = errorIdentifier + this.stage = stage + this.plainName = plainName + this.expectedClass = expectedClass + this.foundClass = foundClass + } +} + +/** + * Checks if the given value is of the expected type. If not, an exception is thrown. + * + * @param stage The stage of the argument (input or output) + * @param par The parameter definition + * @param value The value to check + * @param errorIdentifier The identifier to use in the error message + * @return The value, if it is of the expected type + * @throws UnexpectedArgumentTypeException If the value is not of the expected type +*/ +def _checkArgumentType(String stage, Map par, Object value, String errorIdentifier) { + // expectedClass will only be != null if value is not of the expected type + def expectedClass = null + def foundClass = null + + // todo: split if need be + + if (!par.required && value == null) { + expectedClass = null + } else if (par.multiple) { + if (value !instanceof Collection) { + value = [value] + } + + // split strings + value = value.collectMany{ val -> + if (val instanceof String) { + // collect() to ensure that the result is a List and not simply an array + val.split(par.multiple_sep).collect() + } else { + [val] + } + } + + // process globs + if (par.type == "file" && par.direction == "input") { + value = value.collect{ it instanceof String ? file(it, hidden: true) : it }.flatten() + } + + // check types of elements in list + try { + value = value.collect { listVal -> + _checkArgumentType(stage, par + [multiple: false], listVal, errorIdentifier) + } + } catch (UnexpectedArgumentTypeException e) { + expectedClass = "List[${e.expectedClass}]" + foundClass = "List[${e.foundClass}]" + } + } else if (par.type == "string") { + // cast to string if need be + if (value instanceof GString) { + value = value.toString() + } + expectedClass = value instanceof String ? null : "String" + } else if (par.type == "integer") { + // cast to integer if need be + if (value instanceof String) { + try { + value = value.toInteger() + } catch (NumberFormatException e) { + // do nothing + } + } + if (value instanceof java.math.BigInteger) { + value = value.intValue() + } + expectedClass = value instanceof Integer ? null : "Integer" + } else if (par.type == "long") { + // cast to long if need be + if (value instanceof String) { + try { + value = value.toLong() + } catch (NumberFormatException e) { + // do nothing + } + } + if (value instanceof Integer) { + value = value.toLong() + } + expectedClass = value instanceof Long ? null : "Long" + } else if (par.type == "double") { + // cast to double if need be + if (value instanceof String) { + try { + value = value.toDouble() + } catch (NumberFormatException e) { + // do nothing + } + } + if (value instanceof java.math.BigDecimal) { + value = value.doubleValue() + } + if (value instanceof Float) { + value = value.toDouble() + } + expectedClass = value instanceof Double ? null : "Double" + } else if (par.type == "boolean" | par.type == "boolean_true" | par.type == "boolean_false") { + // cast to boolean if need be + if (value instanceof String) { + def valueLower = value.toLowerCase() + if (valueLower == "true") { + value = true + } else if (valueLower == "false") { + value = false + } + } + expectedClass = value instanceof Boolean ? null : "Boolean" + } else if (par.type == "file" && (par.direction == "input" || stage == "output")) { + // cast to path if need be + if (value instanceof String) { + value = file(value, hidden: true) + } + if (value instanceof File) { + value = value.toPath() + } + expectedClass = value instanceof Path ? null : "Path" + } else if (par.type == "file" && stage == "input" && par.direction == "output") { + // cast to string if need be + if (value instanceof GString) { + value = value.toString() + } + expectedClass = value instanceof String ? null : "String" + } else { + // didn't find a match for par.type + expectedClass = par.type + } + + if (expectedClass != null) { + if (foundClass == null) { + foundClass = value.getClass().getName() + } + throw new UnexpectedArgumentTypeException(errorIdentifier, stage, par.plainName, expectedClass, foundClass) + } + + return value +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_processInputValues.nf' +Map _processInputValues(Map inputs, Map config, String id, String key) { + if (!workflow.stubRun) { + config.allArguments.each { arg -> + if (arg.required) { + assert inputs.containsKey(arg.plainName) && inputs.get(arg.plainName) != null : + "Error in module '${key}' id '${id}': required input argument '${arg.plainName}' is missing" + } + } + + inputs = inputs.collectEntries { name, value -> + def par = config.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") } + assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid input argument" + + value = _checkArgumentType("input", par, value, "in module '$key' id '$id'") + + [ name, value ] + } + } + return inputs +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_processOutputValues.nf' +Map _processOutputValues(Map outputs, Map config, String id, String key) { + if (!workflow.stubRun) { + config.allArguments.each { arg -> + if (arg.direction == "output" && arg.required) { + assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : + "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing" + } + } + + outputs = outputs.collectEntries { name, value -> + def par = config.allArguments.find { it.plainName == name && it.direction == "output" } + assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid output argument" + + value = _checkArgumentType("output", par, value, "in module '$key' id '$id'") + + [ name, value ] + } + } + return outputs +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/IDChecker.nf' +class IDChecker { + final def items = [] as Set + + @groovy.transform.WithWriteLock + boolean observe(String item) { + if (items.contains(item)) { + return false + } else { + items << item + return true + } + } + + @groovy.transform.WithReadLock + boolean contains(String item) { + return items.contains(item) + } + + @groovy.transform.WithReadLock + Set getItems() { + return items.clone() + } +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_checkUniqueIds.nf' + +/** + * Check if the ids are unique across parameter sets + * + * @param parameterSets a list of parameter sets. + */ +private void _checkUniqueIds(List>> parameterSets) { + def ppIds = parameterSets.collect{it[0]} + assert ppIds.size() == ppIds.unique().size() : "All argument sets should have unique ids. Detected ids: $ppIds" +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_getChild.nf' + +// helper functions for reading params from file // +def _getChild(parent, child) { + if (child.contains("://") || java.nio.file.Paths.get(child).isAbsolute()) { + child + } else { + def parentAbsolute = java.nio.file.Paths.get(parent).toAbsolutePath().toString() + parentAbsolute.replaceAll('/[^/]*$', "/") + child + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_parseParamList.nf' +/** + * Figure out the param list format based on the file extension + * + * @param param_list A String containing the path to the parameter list file. + * + * @return A String containing the format of the parameter list file. + */ +def _paramListGuessFormat(param_list) { + if (param_list !instanceof String) { + "asis" + } else if (param_list.endsWith(".csv")) { + "csv" + } else if (param_list.endsWith(".json") || param_list.endsWith(".jsn")) { + "json" + } else if (param_list.endsWith(".yaml") || param_list.endsWith(".yml")) { + "yaml" + } else { + "yaml_blob" + } +} + + +/** + * Read the param list + * + * @param param_list One of the following: + * - A String containing the path to the parameter list file (csv, json or yaml), + * - A yaml blob of a list of maps (yaml_blob), + * - Or a groovy list of maps (asis). + * @param config A Map of the Viash configuration. + * + * @return A List of Maps containing the parameters. + */ +def _parseParamList(param_list, Map config) { + // first determine format by extension + def paramListFormat = _paramListGuessFormat(param_list) + + def paramListPath = (paramListFormat != "asis" && paramListFormat != "yaml_blob") ? + file(param_list, hidden: true) : + null + + // get the correct parser function for the detected params_list format + def paramSets = [] + if (paramListFormat == "asis") { + paramSets = param_list + } else if (paramListFormat == "yaml_blob") { + paramSets = readYamlBlob(param_list) + } else if (paramListFormat == "yaml") { + paramSets = readYaml(paramListPath) + } else if (paramListFormat == "json") { + paramSets = readJson(paramListPath) + } else if (paramListFormat == "csv") { + paramSets = readCsv(paramListPath) + } else { + error "Format of provided --param_list not recognised.\n" + + "Found: '$paramListFormat'.\n" + + "Expected: a csv file, a json file, a yaml file,\n" + + "a yaml blob or a groovy list of maps." + } + + // data checks + assert paramSets instanceof List: "--param_list should contain a list of maps" + for (value in paramSets) { + assert value instanceof Map: "--param_list should contain a list of maps" + } + + // id is argument + def idIsArgument = config.allArguments.any{it.plainName == "id"} + + // Reformat from List to List> by adding the ID as first element of a Tuple2 + paramSets = paramSets.collect({ data -> + def id = data.id + if (!idIsArgument) { + data = data.findAll{k, v -> k != "id"} + } + [id, data] + }) + + // Split parameters with 'multiple: true' + paramSets = paramSets.collect({ id, data -> + data = _splitParams(data, config) + [id, data] + }) + + // The paths of input files inside a param_list file may have been specified relatively to the + // location of the param_list file. These paths must be made absolute. + if (paramListPath) { + paramSets = paramSets.collect({ id, data -> + def new_data = data.collectEntries{ parName, parValue -> + def par = config.allArguments.find{it.plainName == parName} + if (par && par.type == "file" && par.direction == "input") { + if (parValue instanceof Collection) { + parValue = parValue.collectMany{path -> + def x = _resolveSiblingIfNotAbsolute(path, paramListPath) + x instanceof Collection ? x : [x] + } + } else { + parValue = _resolveSiblingIfNotAbsolute(parValue, paramListPath) + } + } + [parName, parValue] + } + [id, new_data] + }) + } + + return paramSets +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_splitParams.nf' +/** + * Split parameters for arguments that accept multiple values using their separator + * + * @param paramList A Map containing parameters to split. + * @param config A Map of the Viash configuration. This Map can be generated from the config file + * using the readConfig() function. + * + * @return A Map of parameters where the parameter values have been split into a list using + * their seperator. + */ +Map _splitParams(Map parValues, Map config){ + def parsedParamValues = parValues.collectEntries { parName, parValue -> + def parameterSettings = config.allArguments.find({it.plainName == parName}) + + if (!parameterSettings) { + // if argument is not found, do not alter + return [parName, parValue] + } + if (parameterSettings.multiple) { // Check if parameter can accept multiple values + if (parValue instanceof Collection) { + parValue = parValue.collect{it instanceof String ? it.split(parameterSettings.multiple_sep) : it } + } else if (parValue instanceof String) { + parValue = parValue.split(parameterSettings.multiple_sep) + } else if (parValue == null) { + parValue = [] + } else { + parValue = [ parValue ] + } + parValue = parValue.flatten() + } + // For all parameters check if multiple values are only passed for + // arguments that allow it. Quietly simplify lists of length 1. + if (!parameterSettings.multiple && parValue instanceof Collection) { + assert parValue.size() == 1 : + "Error: argument ${parName} has too many values.\n" + + " Expected amount: 1. Found: ${parValue.size()}" + parValue = parValue[0] + } + [parName, parValue] + } + return parsedParamValues +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/channelFromParams.nf' +/** + * Parse nextflow parameters based on settings defined in a viash config. + * Return a list of parameter sets, each parameter set corresponding to + * an event in a nextflow channel. The output from this function can be used + * with Channel.fromList to create a nextflow channel with Vdsl3 formatted + * events. + * + * This function performs: + * - A filtering of the params which can be found in the config file. + * - Process the params_list argument which allows a user to to initialise + * a Vsdl3 channel with multiple parameter sets. Possible formats are + * csv, json, yaml, or simply a yaml_blob. A csv should have column names + * which correspond to the different arguments of this pipeline. A json or a yaml + * file should be a list of maps, each of which has keys corresponding to the + * arguments of the pipeline. A yaml blob can also be passed directly as a parameter. + * When passing a csv, json or yaml, relative path names are relativized to the + * location of the parameter file. + * - Combine the parameter sets into a vdsl3 Channel. + * + * @param params Input parameters. Can optionaly contain a 'param_list' key that + * provides a list of arguments that can be split up into multiple events + * in the output channel possible formats of param_lists are: a csv file, + * json file, a yaml file or a yaml blob. Each parameters set (event) must + * have a unique ID. + * @param config A Map of the Viash configuration. This Map can be generated from the config file + * using the readConfig() function. + * + * @return A list of parameters with the first element of the event being + * the event ID and the second element containing a map of the parsed parameters. + */ + +private List>> _paramsToParamSets(Map params, Map config){ + // todo: fetch key from run args + def key_ = config.name + + /* parse regular parameters (not in param_list) */ + /*************************************************/ + def globalParams = config.allArguments + .findAll { params.containsKey(it.plainName) } + .collectEntries { [ it.plainName, params[it.plainName] ] } + def globalID = params.get("id", null) + + /* process params_list arguments */ + /*********************************/ + def paramList = params.containsKey("param_list") && params.param_list != null ? + params.param_list : [] + // if (paramList instanceof String) { + // paramList = [paramList] + // } + // def paramSets = paramList.collectMany{ _parseParamList(it, config) } + // TODO: be able to process param_list when it is a list of strings + def paramSets = _parseParamList(paramList, config) + if (paramSets.isEmpty()) { + paramSets = [[null, [:]]] + } + + /* combine arguments into channel */ + /**********************************/ + def processedParams = paramSets.indexed().collect{ index, tup -> + // Process ID + def id = tup[0] ?: globalID + + if (workflow.stubRun && !id) { + // if stub run, explicitly add an id if missing + id = "stub${index}" + } + assert id != null: "Each parameter set should have at least an 'id'" + + // Process params + def parValues = globalParams + tup[1] + // // Remove parameters which are null, if the default is also null + // parValues = parValues.collectEntries{paramName, paramValue -> + // parameterSettings = config.functionality.allArguments.find({it.plainName == paramName}) + // if ( paramValue != null || parameterSettings.get("default", null) != null ) { + // [paramName, paramValue] + // } + // } + parValues = parValues.collectEntries { name, value -> + def par = config.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") } + assert par != null : "Error in module '${key_}' id '${id}': '${name}' is not a valid input argument" + + if (par == null) { + return [:] + } + value = _checkArgumentType("input", par, value, "in module '$key_' id '$id'") + + [ name, value ] + } + + [id, parValues] + } + + // Check if ids (first element of each list) is unique + _checkUniqueIds(processedParams) + return processedParams +} + +/** + * Parse nextflow parameters based on settings defined in a viash config + * and return a nextflow channel. + * + * @param params Input parameters. Can optionaly contain a 'param_list' key that + * provides a list of arguments that can be split up into multiple events + * in the output channel possible formats of param_lists are: a csv file, + * json file, a yaml file or a yaml blob. Each parameters set (event) must + * have a unique ID. + * @param config A Map of the Viash configuration. This Map can be generated from the config file + * using the readConfig() function. + * + * @return A nextflow Channel with events. Events are formatted as a tuple that contains + * first contains the ID of the event and as second element holds a parameter map. + * + * + */ +def channelFromParams(Map params, Map config) { + def processedParams = _paramsToParamSets(params, config) + return Channel.fromList(processedParams) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/checkUniqueIds.nf' +def checkUniqueIds(Map args) { + def stopOnError = args.stopOnError == null ? args.stopOnError : true + + def idChecker = new IDChecker() + + return filter { tup -> + if (!idChecker.observe(tup[0])) { + if (stopOnError) { + error "Duplicate id: ${tup[0]}" + } else { + log.warn "Duplicate id: ${tup[0]}, removing duplicate entry" + return false + } + } + return true + } +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/preprocessInputs.nf' +// This helper file will be deprecated soon +preprocessInputsDeprecationWarningPrinted = false + +def preprocessInputsDeprecationWarning() { + if (!preprocessInputsDeprecationWarningPrinted) { + preprocessInputsDeprecationWarningPrinted = true + System.err.println("Warning: preprocessInputs() is deprecated and will be removed in Viash 0.9.0.") + } +} + +/** + * Generate a nextflow Workflow that allows processing a channel of + * Vdsl3 formatted events and apply a Viash config to them: + * - Gather default parameters from the Viash config and make + * sure that they are correctly formatted (see applyConfig method). + * - Format the input parameters (also using the applyConfig method). + * - Apply the default parameter to the input parameters. + * - Do some assertions: + * ~ Check if the event IDs in the channel are unique. + * + * The events in the channel are formatted as tuples, with the + * first element of the tuples being a unique id of the parameter set, + * and the second element containg the the parameters themselves. + * Optional extra elements of the tuples will be passed to the output as is. + * + * @param args A map that must contain a 'config' key that points + * to a parsed config (see readConfig()). Optionally, a + * 'key' key can be provided which can be used to create a unique + * name for the workflow process. + * + * @return A workflow that allows processing a channel of Vdsl3 formatted events + * and apply a Viash config to them. + */ +def preprocessInputs(Map args) { + preprocessInputsDeprecationWarning() + + def config = args.config + assert config instanceof Map : + "Error in preprocessInputs: config must be a map. " + + "Expected class: Map. Found: config.getClass() is ${config.getClass()}" + def key_ = args.key ?: config.name + + // Get different parameter types (used throughout this function) + def defaultArgs = config.allArguments + .findAll { it.containsKey("default") } + .collectEntries { [ it.plainName, it.default ] } + + map { tup -> + def id = tup[0] + def data = tup[1] + def passthrough = tup.drop(2) + + def new_data = (defaultArgs + data).collectEntries { name, value -> + def par = config.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") } + + if (par != null) { + value = _checkArgumentType("input", par, value, "in module '$key_' id '$id'") + } + + [ name, value ] + } + + [ id, new_data ] + passthrough + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/runComponents.nf' +/** + * Run a list of components on a stream of data. + * + * @param components: list of Viash VDSL3 modules to run + * @param fromState: a closure, a map or a list of keys to extract from the input data. + * If a closure, it will be called with the id, the data and the component config. + * @param toState: a closure, a map or a list of keys to extract from the output data + * If a closure, it will be called with the id, the output data, the old state and the component config. + * @param filter: filter function to apply to the input. + * It will be called with the id, the data and the component config. + * @param id: id to use for the output data + * If a closure, it will be called with the id, the data and the component config. + * @param auto: auto options to pass to the components + * + * @return: a workflow that runs the components + **/ +def runComponents(Map args) { + log.warn("runComponents is deprecated, use runEach instead") + assert args.components: "runComponents should be passed a list of components to run" + + def components_ = args.components + if (components_ !instanceof List) { + components_ = [ components_ ] + } + assert components_.size() > 0: "pass at least one component to runComponents" + + def fromState_ = args.fromState + def toState_ = args.toState + def filter_ = args.filter + def id_ = args.id + + workflow runComponentsWf { + take: input_ch + main: + + // generate one channel per method + out_chs = components_.collect{ comp_ -> + def comp_config = comp_.config + + def filter_ch = filter_ + ? input_ch | filter{tup -> + filter_(tup[0], tup[1], comp_config) + } + : input_ch + def id_ch = id_ + ? filter_ch | map{tup -> + // def new_id = id_(tup[0], tup[1], comp_config) + def new_id = tup[0] + if (id_ instanceof String) { + new_id = id_ + } else if (id_ instanceof Closure) { + new_id = id_(new_id, tup[1], comp_config) + } + [new_id] + tup.drop(1) + } + : filter_ch + def data_ch = id_ch | map{tup -> + def new_data = tup[1] + if (fromState_ instanceof Map) { + new_data = fromState_.collectEntries{ key0, key1 -> + [key0, new_data[key1]] + } + } else if (fromState_ instanceof List) { + new_data = fromState_.collectEntries{ key -> + [key, new_data[key]] + } + } else if (fromState_ instanceof Closure) { + new_data = fromState_(tup[0], new_data, comp_config) + } + tup.take(1) + [new_data] + tup.drop(1) + } + def out_ch = data_ch + | comp_.run( + auto: (args.auto ?: [:]) + [simplifyInput: false, simplifyOutput: false] + ) + def post_ch = toState_ + ? out_ch | map{tup -> + def output = tup[1] + def old_state = tup[2] + def new_state = null + if (toState_ instanceof Map) { + new_state = old_state + toState_.collectEntries{ key0, key1 -> + [key0, output[key1]] + } + } else if (toState_ instanceof List) { + new_state = old_state + toState_.collectEntries{ key -> + [key, output[key]] + } + } else if (toState_ instanceof Closure) { + new_state = toState_(tup[0], output, old_state, comp_config) + } + [tup[0], new_state] + tup.drop(3) + } + : out_ch + + post_ch + } + + // mix all results + output_ch = + (out_chs.size == 1) + ? out_chs[0] + : out_chs[0].mix(*out_chs.drop(1)) + + emit: output_ch + } + + return runComponentsWf +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/runEach.nf' +/** + * Run a list of components on a stream of data. + * + * @param components: list of Viash VDSL3 modules to run + * @param fromState: a closure, a map or a list of keys to extract from the input data. + * If a closure, it will be called with the id, the data and the component itself. + * @param toState: a closure, a map or a list of keys to extract from the output data + * If a closure, it will be called with the id, the output data, the old state and the component itself. + * @param filter: filter function to apply to the input. + * It will be called with the id, the data and the component itself. + * @param id: id to use for the output data + * If a closure, it will be called with the id, the data and the component itself. + * @param auto: auto options to pass to the components + * + * @return: a workflow that runs the components + **/ +def runEach(Map args) { + assert args.components: "runEach should be passed a list of components to run" + + def components_ = args.components + if (components_ !instanceof List) { + components_ = [ components_ ] + } + assert components_.size() > 0: "pass at least one component to runEach" + + def fromState_ = args.fromState + def toState_ = args.toState + def filter_ = args.filter + def runIf_ = args.runIf + def id_ = args.id + + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + + workflow runEachWf { + take: input_ch + main: + + // generate one channel per method + out_chs = components_.collect{ comp_ -> + def filter_ch = filter_ + ? input_ch | filter{tup -> + filter_(tup[0], tup[1], comp_) + } + : input_ch + def id_ch = id_ + ? filter_ch | map{tup -> + def new_id = id_ + if (new_id instanceof Closure) { + new_id = new_id(tup[0], tup[1], comp_) + } + assert new_id instanceof String : "Error in runEach: id should be a String or a Closure that returns a String. Expected: id instanceof String. Found: ${new_id.getClass()}" + [new_id] + tup.drop(1) + } + : filter_ch + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> + def new_data = tup[1] + if (fromState_ instanceof Map) { + new_data = fromState_.collectEntries{ key0, key1 -> + [key0, new_data[key1]] + } + } else if (fromState_ instanceof List) { + new_data = fromState_.collectEntries{ key -> + [key, new_data[key]] + } + } else if (fromState_ instanceof Closure) { + new_data = fromState_(tup[0], new_data, comp_) + } + tup.take(1) + [new_data] + tup.drop(1) + } + def out_ch = data_ch + | comp_.run( + auto: (args.auto ?: [:]) + [simplifyInput: false, simplifyOutput: false] + ) + def post_ch = toState_ + ? out_ch | map{tup -> + def output = tup[1] + def old_state = tup[2] + def new_state = null + if (toState_ instanceof Map) { + new_state = old_state + toState_.collectEntries{ key0, key1 -> + [key0, output[key1]] + } + } else if (toState_ instanceof List) { + new_state = old_state + toState_.collectEntries{ key -> + [key, output[key]] + } + } else if (toState_ instanceof Closure) { + new_state = toState_(tup[0], output, old_state, comp_) + } + [tup[0], new_state] + tup.drop(3) + } + : out_ch + + def return_ch = post_ch + | concat(chPassthrough) + + return_ch + } + + // mix all results + output_ch = + (out_chs.size == 1) + ? out_chs[0] + : out_chs[0].mix(*out_chs.drop(1)) + + emit: output_ch + } + + return runEachWf +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/safeJoin.nf' +/** + * Join sourceChannel to targetChannel + * + * This function joins the sourceChannel to the targetChannel. + * However, each id in the targetChannel must be present in the + * sourceChannel. If _meta.join_id exists in the targetChannel, that is + * used as an id instead. If the id doesn't match any id in the sourceChannel, + * an error is thrown. + */ + +def safeJoin(targetChannel, sourceChannel, key) { + def sourceIDs = new IDChecker() + + def sourceCheck = sourceChannel + | map { tup -> + sourceIDs.observe(tup[0]) + tup + } + def targetCheck = targetChannel + | map { tup -> + def id = tup[0] + + if (!sourceIDs.contains(id)) { + error ( + "Error in module '${key}' when merging output with original state.\n" + + " Reason: output with id '${id}' could not be joined with source channel.\n" + + " If the IDs in the output channel differ from the input channel,\n" + + " please set `tup[1]._meta.join_id to the original ID.\n" + + " Original IDs in input channel: ['${sourceIDs.getItems().join("', '")}'].\n" + + " Unexpected ID in the output channel: '${id}'.\n" + + " Example input event: [\"id\", [input: file(...)]],\n" + + " Example output event: [\"newid\", [output: file(...), _meta: [join_id: \"id\"]]]" + ) + } + // TODO: add link to our documentation on how to fix this + + tup + } + + sourceCheck.cross(targetChannel) + | map{ left, right -> + right + left.drop(1) + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/config/_processArgument.nf' +def _processArgument(arg) { + arg.multiple = arg.multiple != null ? arg.multiple : false + arg.required = arg.required != null ? arg.required : false + arg.direction = arg.direction != null ? arg.direction : "input" + arg.multiple_sep = arg.multiple_sep != null ? arg.multiple_sep : ";" + arg.plainName = arg.name.replaceAll("^-*", "") + + if (arg.type == "file") { + arg.must_exist = arg.must_exist != null ? arg.must_exist : true + arg.create_parent = arg.create_parent != null ? arg.create_parent : true + } + + // add default values to output files which haven't already got a default + if (arg.type == "file" && arg.direction == "output" && arg.default == null) { + def mult = arg.multiple ? "_*" : "" + def extSearch = "" + if (arg.default != null) { + extSearch = arg.default + } else if (arg.example != null) { + extSearch = arg.example + } + if (extSearch instanceof List) { + extSearch = extSearch[0] + } + def extSearchResult = extSearch.find("\\.[^\\.]+\$") + def ext = extSearchResult != null ? extSearchResult : "" + arg.default = "\$id.\$key.${arg.plainName}${mult}${ext}" + if (arg.multiple) { + arg.default = [arg.default] + } + } + + if (!arg.multiple) { + if (arg.default != null && arg.default instanceof List) { + arg.default = arg.default[0] + } + if (arg.example != null && arg.example instanceof List) { + arg.example = arg.example[0] + } + } + + if (arg.type == "boolean_true") { + arg.default = false + } + if (arg.type == "boolean_false") { + arg.default = true + } + + arg +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/config/addGlobalParams.nf' +def addGlobalArguments(config) { + def localConfig = [ + "argument_groups": [ + [ + "name": "Nextflow input-output arguments", + "description": "Input/output parameters for Nextflow itself. Please note that both publishDir and publish_dir are supported but at least one has to be configured.", + "arguments" : [ + [ + 'name': '--publish_dir', + 'required': true, + 'type': 'string', + 'description': 'Path to an output directory.', + 'example': 'output/', + 'multiple': false + ], + [ + 'name': '--param_list', + 'required': false, + 'type': 'string', + 'description': '''Allows inputting multiple parameter sets to initialise a Nextflow channel. A `param_list` can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. + | + |* A list of maps (as-is) where the keys of each map corresponds to the arguments of the pipeline. Example: in a `nextflow.config` file: `param_list: [ ['id': 'foo', 'input': 'foo.txt'], ['id': 'bar', 'input': 'bar.txt'] ]`. + |* A csv file should have column names which correspond to the different arguments of this pipeline. Example: `--param_list data.csv` with columns `id,input`. + |* A json or a yaml file should be a list of maps, each of which has keys corresponding to the arguments of the pipeline. Example: `--param_list data.json` with contents `[ {'id': 'foo', 'input': 'foo.txt'}, {'id': 'bar', 'input': 'bar.txt'} ]`. + |* A yaml blob can also be passed directly as a string. Example: `--param_list "[ {'id': 'foo', 'input': 'foo.txt'}, {'id': 'bar', 'input': 'bar.txt'} ]"`. + | + |When passing a csv, json or yaml file, relative path names are relativized to the location of the parameter file. No relativation is performed when `param_list` is a list of maps (as-is) or a yaml blob.'''.stripMargin(), + 'example': 'my_params.yaml', + 'multiple': false, + 'hidden': true + ] + // TODO: allow multiple: true in param_list? + // TODO: allow to specify a --param_list_regex to filter the param_list? + // TODO: allow to specify a --param_list_from_state to remap entries in the param_list? + ] + ] + ] + ] + + return processConfig(_mergeMap(config, localConfig)) +} + +def _mergeMap(Map lhs, Map rhs) { + return rhs.inject(lhs.clone()) { map, entry -> + if (map[entry.key] instanceof Map && entry.value instanceof Map) { + map[entry.key] = _mergeMap(map[entry.key], entry.value) + } else if (map[entry.key] instanceof Collection && entry.value instanceof Collection) { + map[entry.key] += entry.value + } else { + map[entry.key] = entry.value + } + return map + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/config/generateHelp.nf' +def _generateArgumentHelp(param) { + // alternatives are not supported + // def names = param.alternatives ::: List(param.name) + + def unnamedProps = [ + ["required parameter", param.required], + ["multiple values allowed", param.multiple], + ["output", param.direction.toLowerCase() == "output"], + ["file must exist", param.type == "file" && param.must_exist] + ].findAll{it[1]}.collect{it[0]} + + def dflt = null + if (param.default != null) { + if (param.default instanceof List) { + dflt = param.default.join(param.multiple_sep != null ? param.multiple_sep : ", ") + } else { + dflt = param.default.toString() + } + } + def example = null + if (param.example != null) { + if (param.example instanceof List) { + example = param.example.join(param.multiple_sep != null ? param.multiple_sep : ", ") + } else { + example = param.example.toString() + } + } + def min = param.min?.toString() + def max = param.max?.toString() + + def escapeChoice = { choice -> + def s1 = choice.replaceAll("\\n", "\\\\n") + def s2 = s1.replaceAll("\"", """\\\"""") + s2.contains(",") || s2 != choice ? "\"" + s2 + "\"" : s2 + } + def choices = param.choices == null ? + null : + "[ " + param.choices.collect{escapeChoice(it.toString())}.join(", ") + " ]" + + def namedPropsStr = [ + ["type", ([param.type] + unnamedProps).join(", ")], + ["default", dflt], + ["example", example], + ["choices", choices], + ["min", min], + ["max", max] + ] + .findAll{it[1]} + .collect{"\n " + it[0] + ": " + it[1].replaceAll("\n", "\\n")} + .join("") + + def descStr = param.description == null ? + "" : + _paragraphWrap("\n" + param.description.trim(), 80 - 8).join("\n ") + + "\n --" + param.plainName + + namedPropsStr + + descStr +} + +// Based on Helper.generateHelp() in Helper.scala +def _generateHelp(config) { + def fun = config + + // PART 1: NAME AND VERSION + def nameStr = fun.name + + (fun.version == null ? "" : " " + fun.version) + + // PART 2: DESCRIPTION + def descrStr = fun.description == null ? + "" : + "\n\n" + _paragraphWrap(fun.description.trim(), 80).join("\n") + + // PART 3: Usage + def usageStr = fun.usage == null ? + "" : + "\n\nUsage:\n" + fun.usage.trim() + + // PART 4: Options + def argGroupStrs = fun.allArgumentGroups.collect{argGroup -> + def name = argGroup.name + def descriptionStr = argGroup.description == null ? + "" : + "\n " + _paragraphWrap(argGroup.description.trim(), 80-4).join("\n ") + "\n" + def arguments = argGroup.arguments.collect{arg -> + arg instanceof String ? fun.allArguments.find{it.plainName == arg} : arg + }.findAll{it != null} + def argumentStrs = arguments.collect{param -> _generateArgumentHelp(param)} + + "\n\n$name:" + + descriptionStr + + argumentStrs.join("\n") + } + + // FINAL: combine + def out = nameStr + + descrStr + + usageStr + + argGroupStrs.join("") + + return out +} + +// based on Format._paragraphWrap +def _paragraphWrap(str, maxLength) { + def outLines = [] + str.split("\n").each{par -> + def words = par.split("\\s").toList() + + def word = null + def line = words.pop() + while(!words.isEmpty()) { + word = words.pop() + if (line.length() + word.length() + 1 <= maxLength) { + line = line + " " + word + } else { + outLines.add(line) + line = word + } + } + if (words.isEmpty()) { + outLines.add(line) + } + } + return outLines +} + +def helpMessage(config) { + if (params.containsKey("help") && params.help) { + def mergedConfig = addGlobalArguments(config) + def helpStr = _generateHelp(mergedConfig) + println(helpStr) + exit 0 + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/config/processConfig.nf' +def processConfig(config) { + // set defaults for arguments + config.arguments = + (config.arguments ?: []).collect{_processArgument(it)} + + // set defaults for argument_group arguments + config.argument_groups = + (config.argument_groups ?: []).collect{grp -> + grp.arguments = (grp.arguments ?: []).collect{_processArgument(it)} + grp + } + + // create combined arguments list + config.allArguments = + config.arguments + + config.argument_groups.collectMany{it.arguments} + + // add missing argument groups (based on Functionality::allArgumentGroups()) + def argGroups = config.argument_groups + if (argGroups.any{it.name.toLowerCase() == "arguments"}) { + argGroups = argGroups.collect{ grp -> + if (grp.name.toLowerCase() == "arguments") { + grp = grp + [ + arguments: grp.arguments + config.arguments + ] + } + grp + } + } else { + argGroups = argGroups + [ + name: "Arguments", + arguments: config.arguments + ] + } + config.allArgumentGroups = argGroups + + config +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/config/readConfig.nf' + +def readConfig(file) { + def config = readYaml(file ?: moduleDir.resolve("config.vsh.yaml")) + processConfig(config) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/_resolveSiblingIfNotAbsolute.nf' +/** + * Resolve a path relative to the current file. + * + * @param str The path to resolve, as a String. + * @param parentPath The path to resolve relative to, as a Path. + * + * @return The path that may have been resovled, as a Path. + */ +def _resolveSiblingIfNotAbsolute(str, parentPath) { + if (str !instanceof String) { + return str + } + if (!_stringIsAbsolutePath(str)) { + return parentPath.resolveSibling(str) + } else { + return file(str, hidden: true) + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/_stringIsAbsolutePath.nf' +/** + * Check whether a path as a string is absolute. + * + * In the past, we tried using `file(., relative: true).isAbsolute()`, + * but the 'relative' option was added in 22.10.0. + * + * @param path The path to check, as a String. + * + * @return Whether the path is absolute, as a boolean. + */ +def _stringIsAbsolutePath(path) { + def _resolve_URL_PROTOCOL = ~/^([a-zA-Z][a-zA-Z0-9]*:)?\\/.+/ + + assert path instanceof String + return _resolve_URL_PROTOCOL.matcher(path).matches() +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/collectTraces.nf' +class CustomTraceObserver implements nextflow.trace.TraceObserver { + List traces + + CustomTraceObserver(List traces) { + this.traces = traces + } + + @Override + void onProcessComplete(nextflow.processor.TaskHandler handler, nextflow.trace.TraceRecord trace) { + def trace2 = trace.store.clone() + trace2.script = null + traces.add(trace2) + } + + @Override + void onProcessCached(nextflow.processor.TaskHandler handler, nextflow.trace.TraceRecord trace) { + def trace2 = trace.store.clone() + trace2.script = null + traces.add(trace2) + } +} + +def collectTraces() { + def traces = Collections.synchronizedList([]) + + // add custom trace observer which stores traces in the traces object + session.observers.add(new CustomTraceObserver(traces)) + + traces +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/deepClone.nf' +/** + * Performs a deep clone of the given object. + * @param x an object + */ +def deepClone(x) { + iterateMap(x, {it instanceof Cloneable ? it.clone() : it}) +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/getPublishDir.nf' +def getPublishDir() { + return params.containsKey("publish_dir") ? params.publish_dir : + params.containsKey("publishDir") ? params.publishDir : + null +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/getRootDir.nf' + +// Recurse upwards until we find a '.build.yaml' file +def _findBuildYamlFile(pathPossiblySymlink) { + def path = pathPossiblySymlink.toRealPath() + def child = path.resolve(".build.yaml") + if (java.nio.file.Files.isDirectory(path) && java.nio.file.Files.exists(child)) { + return child + } else { + def parent = path.getParent() + if (parent == null) { + return null + } else { + return _findBuildYamlFile(parent) + } + } +} + +// get the root of the target folder +def getRootDir() { + def dir = _findBuildYamlFile(meta.resources_dir) + assert dir != null: "Could not find .build.yaml in the folder structure" + dir.getParent() +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/iterateMap.nf' +/** + * Recursively apply a function over the leaves of an object. + * @param obj The object to iterate over. + * @param fun The function to apply to each value. + * @return The object with the function applied to each value. + */ +def iterateMap(obj, fun) { + if (obj instanceof List && obj !instanceof String) { + return obj.collect{item -> + iterateMap(item, fun) + } + } else if (obj instanceof Map) { + return obj.collectEntries{key, item -> + [key.toString(), iterateMap(item, fun)] + } + } else { + return fun(obj) + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/niceView.nf' +/** + * A view for printing the event of each channel as a YAML blob. + * This is useful for debugging. + */ +def niceView() { + workflow niceViewWf { + take: input + main: + output = input + | view{toYamlBlob(it)} + emit: output + } + return niceViewWf +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readCsv.nf' + +def readCsv(file_path) { + def output = [] + def inputFile = file_path !instanceof Path ? file(file_path, hidden: true) : file_path + + // todo: allow escaped quotes in string + // todo: allow single quotes? + def splitRegex = java.util.regex.Pattern.compile(''',(?=(?:[^"]*"[^"]*")*[^"]*$)''') + def removeQuote = java.util.regex.Pattern.compile('''"(.*)"''') + + def br = java.nio.file.Files.newBufferedReader(inputFile) + + def row = -1 + def header = null + while (br.ready() && header == null) { + def line = br.readLine() + row++ + if (!line.startsWith("#")) { + header = splitRegex.split(line, -1).collect{field -> + m = removeQuote.matcher(field) + m.find() ? m.replaceFirst('$1') : field + } + } + } + assert header != null: "CSV file should contain a header" + + while (br.ready()) { + def line = br.readLine() + row++ + if (line == null) { + br.close() + break + } + + if (!line.startsWith("#")) { + def predata = splitRegex.split(line, -1) + def data = predata.collect{field -> + if (field == "") { + return null + } + def m = removeQuote.matcher(field) + if (m.find()) { + return m.replaceFirst('$1') + } else { + return field + } + } + assert header.size() == data.size(): "Row $row should contain the same number as fields as the header" + + def dataMap = [header, data].transpose().collectEntries().findAll{it.value != null} + output.add(dataMap) + } + } + + output +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readJson.nf' +def readJson(file_path) { + def inputFile = file_path !instanceof Path ? file(file_path, hidden: true) : file_path + def jsonSlurper = new groovy.json.JsonSlurper() + jsonSlurper.parse(inputFile) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readJsonBlob.nf' +def readJsonBlob(str) { + def jsonSlurper = new groovy.json.JsonSlurper() + jsonSlurper.parseText(str) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readTaggedYaml.nf' +// Custom constructor to modify how certain objects are parsed from YAML +class CustomConstructor extends org.yaml.snakeyaml.constructor.Constructor { + Path root + + class ConstructPath extends org.yaml.snakeyaml.constructor.AbstractConstruct { + public Object construct(org.yaml.snakeyaml.nodes.Node node) { + String filename = (String) constructScalar(node); + if (root != null) { + return root.resolve(filename); + } + return java.nio.file.Paths.get(filename); + } + } + + CustomConstructor(org.yaml.snakeyaml.LoaderOptions options, Path root) { + super(options) + this.root = root + // Handling !file tag and parse it back to a File type + this.yamlConstructors.put(new org.yaml.snakeyaml.nodes.Tag("!file"), new ConstructPath()) + } +} + +def readTaggedYaml(Path path) { + def options = new org.yaml.snakeyaml.LoaderOptions() + def constructor = new CustomConstructor(options, path.getParent()) + def yaml = new org.yaml.snakeyaml.Yaml(constructor) + return yaml.load(path.text) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readYaml.nf' +def readYaml(file_path) { + def inputFile = file_path !instanceof Path ? file(file_path, hidden: true) : file_path + def yamlSlurper = new org.yaml.snakeyaml.Yaml() + yamlSlurper.load(inputFile) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readYamlBlob.nf' +def readYamlBlob(str) { + def yamlSlurper = new org.yaml.snakeyaml.Yaml() + yamlSlurper.load(str) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/toJsonBlob.nf' +String toJsonBlob(data) { + return groovy.json.JsonOutput.toJson(data) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/toTaggedYamlBlob.nf' +// Custom representer to modify how certain objects are represented in YAML +class CustomRepresenter extends org.yaml.snakeyaml.representer.Representer { + Path relativizer + + class RepresentPath implements org.yaml.snakeyaml.representer.Represent { + public String getFileName(Object obj) { + if (obj instanceof File) { + obj = ((File) obj).toPath(); + } + if (obj !instanceof Path) { + throw new IllegalArgumentException("Object: " + obj + " is not a Path or File"); + } + def path = (Path) obj; + + if (relativizer != null) { + return relativizer.relativize(path).toString() + } else { + return path.toString() + } + } + + public org.yaml.snakeyaml.nodes.Node representData(Object data) { + String filename = getFileName(data); + def tag = new org.yaml.snakeyaml.nodes.Tag("!file"); + return representScalar(tag, filename); + } + } + CustomRepresenter(org.yaml.snakeyaml.DumperOptions options, Path relativizer) { + super(options) + this.relativizer = relativizer + this.representers.put(sun.nio.fs.UnixPath, new RepresentPath()) + this.representers.put(Path, new RepresentPath()) + this.representers.put(File, new RepresentPath()) + } +} + +String toTaggedYamlBlob(data) { + return toRelativeTaggedYamlBlob(data, null) +} +String toRelativeTaggedYamlBlob(data, Path relativizer) { + def options = new org.yaml.snakeyaml.DumperOptions() + options.setDefaultFlowStyle(org.yaml.snakeyaml.DumperOptions.FlowStyle.BLOCK) + def representer = new CustomRepresenter(options, relativizer) + def yaml = new org.yaml.snakeyaml.Yaml(representer, options) + return yaml.dump(data) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/toYamlBlob.nf' +String toYamlBlob(data) { + def options = new org.yaml.snakeyaml.DumperOptions() + options.setDefaultFlowStyle(org.yaml.snakeyaml.DumperOptions.FlowStyle.BLOCK) + options.setPrettyFlow(true) + def yaml = new org.yaml.snakeyaml.Yaml(options) + def cleanData = iterateMap(data, { it instanceof Path ? it.toString() : it }) + return yaml.dump(cleanData) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/writeJson.nf' +void writeJson(data, file) { + assert data: "writeJson: data should not be null" + assert file: "writeJson: file should not be null" + file.write(toJsonBlob(data)) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/writeYaml.nf' +void writeYaml(data, file) { + assert data: "writeYaml: data should not be null" + assert file: "writeYaml: file should not be null" + file.write(toYamlBlob(data)) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/findStates.nf' +def findStates(Map params, Map config) { + def auto_config = deepClone(config) + def auto_params = deepClone(params) + + auto_config = auto_config.clone() + // override arguments + auto_config.argument_groups = [] + auto_config.arguments = [ + [ + type: "string", + name: "--id", + description: "A dummy identifier", + required: false + ], + [ + type: "file", + name: "--input_states", + example: "/path/to/input/directory/**/state.yaml", + description: "Path to input directory containing the datasets to be integrated.", + required: true, + multiple: true, + multiple_sep: ";" + ], + [ + type: "string", + name: "--filter", + example: "foo/.*/state.yaml", + description: "Regex to filter state files by path.", + required: false + ], + // to do: make this a yaml blob? + [ + type: "string", + name: "--rename_keys", + example: ["newKey1:oldKey1", "newKey2:oldKey2"], + description: "Rename keys in the detected input files. This is useful if the input files do not match the set of input arguments of the workflow.", + required: false, + multiple: true, + multiple_sep: ";" + ], + [ + type: "string", + name: "--settings", + example: '{"output_dataset": "dataset.h5ad", "k": 10}', + description: "Global arguments as a JSON glob to be passed to all components.", + required: false + ] + ] + if (!(auto_params.containsKey("id"))) { + auto_params["id"] = "auto" + } + + // run auto config through processConfig once more + auto_config = processConfig(auto_config) + + workflow findStatesWf { + helpMessage(auto_config) + + output_ch = + channelFromParams(auto_params, auto_config) + | flatMap { autoId, args -> + + def globalSettings = args.settings ? readYamlBlob(args.settings) : [:] + + // look for state files in input dir + def stateFiles = args.input_states + + // filter state files by regex + if (args.filter) { + stateFiles = stateFiles.findAll{ stateFile -> + def stateFileStr = stateFile.toString() + def matcher = stateFileStr =~ args.filter + matcher.matches()} + } + + // read in states + def states = stateFiles.collect { stateFile -> + def state_ = readTaggedYaml(stateFile) + [state_.id, state_] + } + + // construct renameMap + if (args.rename_keys) { + def renameMap = args.rename_keys.collectEntries{renameString -> + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" + split + } + + // rename keys in state, only let states through which have all keys + // also add global settings + states = states.collectMany{id, state -> + def newState = [:] + + for (key in renameMap.keySet()) { + def origKey = renameMap[key] + if (!(state.containsKey(origKey))) { + return [] + } + newState[key] = state[origKey] + } + + [[id, globalSettings + newState]] + } + } + + states + } + emit: + output_ch + } + + return findStatesWf +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/joinStates.nf' +def joinStates(Closure apply_) { + workflow joinStatesWf { + take: input_ch + main: + output_ch = input_ch + | toSortedList + | filter{ it.size() > 0 } + | map{ tups -> + def ids = tups.collect{it[0]} + def states = tups.collect{it[1]} + apply_(ids, states) + } + + emit: output_ch + } + return joinStatesWf +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishStates.nf' +def collectFiles(obj) { + if (obj instanceof java.io.File || obj instanceof Path) { + return [obj] + } else if (obj instanceof List && obj !instanceof String) { + return obj.collectMany{item -> + collectFiles(item) + } + } else if (obj instanceof Map) { + return obj.collectMany{key, item -> + collectFiles(item) + } + } else { + return [] + } +} + +/** + * Recurse through a state and collect all input files and their target output filenames. + * @param obj The state to recurse through. + * @param prefix The prefix to prepend to the output filenames. + */ +def collectInputOutputPaths(obj, prefix) { + if (obj instanceof File || obj instanceof Path) { + def path = obj instanceof Path ? obj : obj.toPath() + def ext = path.getFileName().toString().find("\\.[^\\.]+\$") ?: "" + def newFilename = prefix + ext + return [[obj, newFilename]] + } else if (obj instanceof List && obj !instanceof String) { + return obj.withIndex().collectMany{item, ix -> + collectInputOutputPaths(item, prefix + "_" + ix) + } + } else if (obj instanceof Map) { + return obj.collectMany{key, item -> + collectInputOutputPaths(item, prefix + "." + key) + } + } else { + return [] + } +} + +def publishStates(Map args) { + def key_ = args.get("key") + def yamlTemplate_ = args.get("output_state", args.get("outputState", '$id.$key.state.yaml')) + + assert key_ != null : "publishStates: key must be specified" + + workflow publishStatesWf { + take: input_ch + main: + input_ch + | map { tup -> + def id_ = tup[0] + def state_ = tup[1] + + // the input files and the target output filenames + def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose() + def inputFiles_ = inputoutputFilenames_[0] + def outputFilenames_ = inputoutputFilenames_[1] + + def yamlFilename = yamlTemplate_ + .replaceAll('\\$id', id_) + .replaceAll('\\$key', key_) + + // TODO: do the pathnames in state_ match up with the outputFilenames_? + + // convert state to yaml blob + def yamlBlob_ = toRelativeTaggedYamlBlob([id: id_] + state_, java.nio.file.Paths.get(yamlFilename)) + + [id_, yamlBlob_, yamlFilename, inputFiles_, outputFilenames_] + } + | publishStatesProc + emit: input_ch + } + return publishStatesWf +} +process publishStatesProc { + // todo: check publishpath? + publishDir path: "${getPublishDir()}/", mode: "copy" + tag "$id" + input: + tuple val(id), val(yamlBlob), val(yamlFile), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles) + output: + tuple val(id), path{[yamlFile] + outputFiles} + script: + def copyCommands = [ + inputFiles instanceof List ? inputFiles : [inputFiles], + outputFiles instanceof List ? outputFiles : [outputFiles] + ] + .transpose() + .collectMany{infile, outfile -> + if (infile.toString() != outfile.toString()) { + [ + "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"", + "cp -r '${infile.toString()}' '${outfile.toString()}'" + ] + } else { + // no need to copy if infile is the same as outfile + [] + } + } + """ +mkdir -p "\$(dirname '${yamlFile}')" +echo "Storing state as yaml" +echo '${yamlBlob}' > '${yamlFile}' +echo "Copying output files to destination folder" +${copyCommands.join("\n ")} +""" +} + + +// this assumes that the state contains no other values other than those specified in the config +def publishStatesByConfig(Map args) { + def config = args.get("config") + assert config != null : "publishStatesByConfig: config must be specified" + + def key_ = args.get("key", config.name) + assert key_ != null : "publishStatesByConfig: key must be specified" + + workflow publishStatesSimpleWf { + take: input_ch + main: + input_ch + | map { tup -> + def id_ = tup[0] + def state_ = tup[1] // e.g. [output: new File("myoutput.h5ad"), k: 10] + def origState_ = tup[2] // e.g. [output: '$id.$key.foo.h5ad'] + + // TODO: allow overriding the state.yaml template + // TODO TODO: if auto.publish == "state", add output_state as an argument + def yamlTemplate = params.containsKey("output_state") ? params.output_state : '$id.$key.state.yaml' + def yamlFilename = yamlTemplate + .replaceAll('\\$id', id_) + .replaceAll('\\$key', key_) + def yamlDir = java.nio.file.Paths.get(yamlFilename).getParent() + + // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where + // - key is a String + // - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path) + // - inputPath is a List[Path] + // - outputFilename is a List[String] + // - (key, value) are the tuples that will be saved to the state.yaml file + // - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml) + def processedState = + config.allArguments + .findAll { it.direction == "output" } + .collectMany { par -> + def plainName_ = par.plainName + // if the state does not contain the key, it's an + // optional argument for which the component did + // not generate any output + if (!state_.containsKey(plainName_)) { + return [] + } + def value = state_[plainName_] + // if the parameter is not a file, it should be stored + // in the state as-is, but is not something that needs + // to be copied from the source path to the dest path + if (par.type != "file") { + return [[key: plainName_, value: value, inputPath: [], outputFilename: []]] + } + // if the orig state does not contain this filename, + // it's an optional argument for which the user specified + // that it should not be returned as a state + if (!origState_.containsKey(plainName_)) { + return [] + } + def filenameTemplate = origState_[plainName_] + // if the pararameter is multiple: true, fetch the template + if (par.multiple && filenameTemplate instanceof List) { + filenameTemplate = filenameTemplate[0] + } + // instantiate the template + def filename = filenameTemplate + .replaceAll('\\$id', id_) + .replaceAll('\\$key', key_) + if (par.multiple) { + // if the parameter is multiple: true, the filename + // should contain a wildcard '*' that is replaced with + // the index of the file + assert filename.contains("*") : "Module '${key_}' id '${id_}': Multiple output files specified, but no wildcard '*' in the filename: ${filename}" + def outputPerFile = value.withIndex().collect{ val, ix -> + def filename_ix = filename.replace("*", ix.toString()) + def value_ = java.nio.file.Paths.get(filename_ix) + // if id contains a slash + if (yamlDir != null) { + value_ = yamlDir.relativize(value_) + } + def inputPath = val instanceof File ? val.toPath() : val + [value: value_, inputPath: inputPath, outputFilename: filename_ix] + } + def transposedOutputs = ["value", "inputPath", "outputFilename"].collectEntries{ key -> + [key, outputPerFile.collect{dic -> dic[key]}] + } + return [[key: plainName_] + transposedOutputs] + } else { + def value_ = java.nio.file.Paths.get(filename) + // if id contains a slash + if (yamlDir != null) { + value_ = yamlDir.relativize(value_) + } + def inputPath = value instanceof File ? value.toPath() : value + return [[key: plainName_, value: value_, inputPath: [inputPath], outputFilename: [filename]]] + } + } + + def updatedState_ = processedState.collectEntries{[it.key, it.value]} + def inputPaths = processedState.collectMany{it.inputPath} + def outputFilenames = processedState.collectMany{it.outputFilename} + + // convert state to yaml blob + def yamlBlob_ = toTaggedYamlBlob([id: id_] + updatedState_) + + [id_, yamlBlob_, yamlFilename, inputPaths, outputFilenames] + } + | publishStatesProc + emit: input_ch + } + return publishStatesSimpleWf +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/setState.nf' +def setState(fun) { + assert fun instanceof Closure || fun instanceof Map || fun instanceof List : + "Error in setState: Expected process argument to be a Closure, a Map, or a List. Found: class ${fun.getClass()}" + + // if fun is a List, convert to map + if (fun instanceof List) { + // check whether fun is a list[string] + assert fun.every{it instanceof CharSequence} : "Error in setState: argument is a List, but not all elements are Strings" + fun = fun.collectEntries{[it, it]} + } + + // if fun is a map, convert to closure + if (fun instanceof Map) { + // check whether fun is a map[string, string] + assert fun.values().every{it instanceof CharSequence} : "Error in setState: argument is a Map, but not all values are Strings" + assert fun.keySet().every{it instanceof CharSequence} : "Error in setState: argument is a Map, but not all keys are Strings" + def funMap = fun.clone() + // turn the map into a closure to be used later on + fun = { id_, state_ -> + assert state_ instanceof Map : "Error in setState: the state is not a Map" + funMap.collectMany{newkey, origkey -> + if (state_.containsKey(origkey)) { + [[newkey, state_[origkey]]] + } else { + [] + } + }.collectEntries() + } + } + + map { tup -> + def id = tup[0] + def state = tup[1] + def unfilteredState = fun(id, state) + def newState = unfilteredState.findAll{key, val -> val != null} + [id, newState] + tup.drop(2) + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/processAuto.nf' +// TODO: unit test processAuto +def processAuto(Map auto) { + // remove null values + auto = auto.findAll{k, v -> v != null} + + // check for unexpected keys + def expectedKeys = ["simplifyInput", "simplifyOutput", "transcript", "publish"] + def unexpectedKeys = auto.keySet() - expectedKeys + assert unexpectedKeys.isEmpty(), "unexpected keys in auto: '${unexpectedKeys.join("', '")}'" + + // check auto.simplifyInput + assert auto.simplifyInput instanceof Boolean, "auto.simplifyInput must be a boolean" + + // check auto.simplifyOutput + assert auto.simplifyOutput instanceof Boolean, "auto.simplifyOutput must be a boolean" + + // check auto.transcript + assert auto.transcript instanceof Boolean, "auto.transcript must be a boolean" + + // check auto.publish + assert auto.publish instanceof Boolean || auto.publish == "state", "auto.publish must be a boolean or 'state'" + + return auto.subMap(expectedKeys) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/processDirectives.nf' +def assertMapKeys(map, expectedKeys, requiredKeys, mapName) { + assert map instanceof Map : "Expected argument '$mapName' to be a Map. Found: class ${map.getClass()}" + map.forEach { key, val -> + assert key in expectedKeys : "Unexpected key '$key' in ${mapName ? mapName + " " : ""}map" + } + requiredKeys.forEach { requiredKey -> + assert map.containsKey(requiredKey) : "Missing required key '$key' in ${mapName ? mapName + " " : ""}map" + } +} + +// TODO: unit test processDirectives +def processDirectives(Map drctv) { + // remove null values + drctv = drctv.findAll{k, v -> v != null} + + // check for unexpected keys + def expectedKeys = [ + "accelerator", "afterScript", "beforeScript", "cache", "conda", "container", "containerOptions", "cpus", "disk", "echo", "errorStrategy", "executor", "machineType", "maxErrors", "maxForks", "maxRetries", "memory", "module", "penv", "pod", "publishDir", "queue", "label", "scratch", "storeDir", "stageInMode", "stageOutMode", "tag", "time" + ] + def unexpectedKeys = drctv.keySet() - expectedKeys + assert unexpectedKeys.isEmpty() : "Unexpected keys in process directive: '${unexpectedKeys.join("', '")}'" + + /* DIRECTIVE accelerator + accepted examples: + - [ limit: 4, type: "nvidia-tesla-k80" ] + */ + if (drctv.containsKey("accelerator")) { + assertMapKeys(drctv["accelerator"], ["type", "limit", "request", "runtime"], [], "accelerator") + } + + /* DIRECTIVE afterScript + accepted examples: + - "source /cluster/bin/cleanup" + */ + if (drctv.containsKey("afterScript")) { + assert drctv["afterScript"] instanceof CharSequence + } + + /* DIRECTIVE beforeScript + accepted examples: + - "source /cluster/bin/setup" + */ + if (drctv.containsKey("beforeScript")) { + assert drctv["beforeScript"] instanceof CharSequence + } + + /* DIRECTIVE cache + accepted examples: + - true + - false + - "deep" + - "lenient" + */ + if (drctv.containsKey("cache")) { + assert drctv["cache"] instanceof CharSequence || drctv["cache"] instanceof Boolean + if (drctv["cache"] instanceof CharSequence) { + assert drctv["cache"] in ["deep", "lenient"] : "Unexpected value for cache" + } + } + + /* DIRECTIVE conda + accepted examples: + - "bwa=0.7.15" + - "bwa=0.7.15 fastqc=0.11.5" + - ["bwa=0.7.15", "fastqc=0.11.5"] + */ + if (drctv.containsKey("conda")) { + if (drctv["conda"] instanceof List) { + drctv["conda"] = drctv["conda"].join(" ") + } + assert drctv["conda"] instanceof CharSequence + } + + /* DIRECTIVE container + accepted examples: + - "foo/bar:tag" + - [ registry: "reg", image: "im", tag: "ta" ] + is transformed to "reg/im:ta" + - [ image: "im" ] + is transformed to "im:latest" + */ + if (drctv.containsKey("container")) { + assert drctv["container"] instanceof Map || drctv["container"] instanceof CharSequence + if (drctv["container"] instanceof Map) { + def m = drctv["container"] + assertMapKeys(m, [ "registry", "image", "tag" ], ["image"], "container") + def part1 = + System.getenv('OVERRIDE_CONTAINER_REGISTRY') ? System.getenv('OVERRIDE_CONTAINER_REGISTRY') + "/" : + params.containsKey("override_container_registry") ? params["override_container_registry"] + "/" : // todo: remove? + m.registry ? m.registry + "/" : + "" + def part2 = m.image + def part3 = m.tag ? ":" + m.tag : ":latest" + drctv["container"] = part1 + part2 + part3 + } + } + + /* DIRECTIVE containerOptions + accepted examples: + - "--foo bar" + - ["--foo bar", "-f b"] + */ + if (drctv.containsKey("containerOptions")) { + if (drctv["containerOptions"] instanceof List) { + drctv["containerOptions"] = drctv["containerOptions"].join(" ") + } + assert drctv["containerOptions"] instanceof CharSequence + } + + /* DIRECTIVE cpus + accepted examples: + - 1 + - 10 + */ + if (drctv.containsKey("cpus")) { + assert drctv["cpus"] instanceof Integer + } + + /* DIRECTIVE disk + accepted examples: + - "1 GB" + - "2TB" + - "3.2KB" + - "10.B" + */ + if (drctv.containsKey("disk")) { + assert drctv["disk"] instanceof CharSequence + // assert drctv["disk"].matches("[0-9]+(\\.[0-9]*)? *[KMGTPEZY]?B") + // ^ does not allow closures + } + + /* DIRECTIVE echo + accepted examples: + - true + - false + */ + if (drctv.containsKey("echo")) { + assert drctv["echo"] instanceof Boolean + } + + /* DIRECTIVE errorStrategy + accepted examples: + - "terminate" + - "finish" + */ + if (drctv.containsKey("errorStrategy")) { + assert drctv["errorStrategy"] instanceof CharSequence + assert drctv["errorStrategy"] in ["terminate", "finish", "ignore", "retry"] : "Unexpected value for errorStrategy" + } + + /* DIRECTIVE executor + accepted examples: + - "local" + - "sge" + */ + if (drctv.containsKey("executor")) { + assert drctv["executor"] instanceof CharSequence + assert drctv["executor"] in ["local", "sge", "uge", "lsf", "slurm", "pbs", "pbspro", "moab", "condor", "nqsii", "ignite", "k8s", "awsbatch", "google-pipelines"] : "Unexpected value for executor" + } + + /* DIRECTIVE machineType + accepted examples: + - "n1-highmem-8" + */ + if (drctv.containsKey("machineType")) { + assert drctv["machineType"] instanceof CharSequence + } + + /* DIRECTIVE maxErrors + accepted examples: + - 1 + - 3 + */ + if (drctv.containsKey("maxErrors")) { + assert drctv["maxErrors"] instanceof Integer + } + + /* DIRECTIVE maxForks + accepted examples: + - 1 + - 3 + */ + if (drctv.containsKey("maxForks")) { + assert drctv["maxForks"] instanceof Integer + } + + /* DIRECTIVE maxRetries + accepted examples: + - 1 + - 3 + */ + if (drctv.containsKey("maxRetries")) { + assert drctv["maxRetries"] instanceof Integer + } + + /* DIRECTIVE memory + accepted examples: + - "1 GB" + - "2TB" + - "3.2KB" + - "10.B" + */ + if (drctv.containsKey("memory")) { + assert drctv["memory"] instanceof CharSequence + // assert drctv["memory"].matches("[0-9]+(\\.[0-9]*)? *[KMGTPEZY]?B") + // ^ does not allow closures + } + + /* DIRECTIVE module + accepted examples: + - "ncbi-blast/2.2.27" + - "ncbi-blast/2.2.27:t_coffee/10.0" + - ["ncbi-blast/2.2.27", "t_coffee/10.0"] + */ + if (drctv.containsKey("module")) { + if (drctv["module"] instanceof List) { + drctv["module"] = drctv["module"].join(":") + } + assert drctv["module"] instanceof CharSequence + } + + /* DIRECTIVE penv + accepted examples: + - "smp" + */ + if (drctv.containsKey("penv")) { + assert drctv["penv"] instanceof CharSequence + } + + /* DIRECTIVE pod + accepted examples: + - [ label: "key", value: "val" ] + - [ annotation: "key", value: "val" ] + - [ env: "key", value: "val" ] + - [ [label: "l", value: "v"], [env: "e", value: "v"]] + */ + if (drctv.containsKey("pod")) { + if (drctv["pod"] instanceof Map) { + drctv["pod"] = [ drctv["pod"] ] + } + assert drctv["pod"] instanceof List + drctv["pod"].forEach { pod -> + assert pod instanceof Map + // TODO: should more checks be added? + // See https://www.nextflow.io/docs/latest/process.html?highlight=directives#pod + // e.g. does it contain 'label' and 'value', or 'annotation' and 'value', or ...? + } + } + + /* DIRECTIVE publishDir + accepted examples: + - [] + - [ [ path: "foo", enabled: true ], [ path: "bar", enabled: false ] ] + - "/path/to/dir" + is transformed to [[ path: "/path/to/dir" ]] + - [ path: "/path/to/dir", mode: "cache" ] + is transformed to [[ path: "/path/to/dir", mode: "cache" ]] + */ + // TODO: should we also look at params["publishDir"]? + if (drctv.containsKey("publishDir")) { + def pblsh = drctv["publishDir"] + + // check different options + assert pblsh instanceof List || pblsh instanceof Map || pblsh instanceof CharSequence + + // turn into list if not already so + // for some reason, 'if (!pblsh instanceof List) pblsh = [ pblsh ]' doesn't work. + pblsh = pblsh instanceof List ? pblsh : [ pblsh ] + + // check elements of publishDir + pblsh = pblsh.collect{ elem -> + // turn into map if not already so + elem = elem instanceof CharSequence ? [ path: elem ] : elem + + // check types and keys + assert elem instanceof Map : "Expected publish argument '$elem' to be a String or a Map. Found: class ${elem.getClass()}" + assertMapKeys(elem, [ "path", "mode", "overwrite", "pattern", "saveAs", "enabled" ], ["path"], "publishDir") + + // check elements in map + assert elem.containsKey("path") + assert elem["path"] instanceof CharSequence + if (elem.containsKey("mode")) { + assert elem["mode"] instanceof CharSequence + assert elem["mode"] in [ "symlink", "rellink", "link", "copy", "copyNoFollow", "move" ] + } + if (elem.containsKey("overwrite")) { + assert elem["overwrite"] instanceof Boolean + } + if (elem.containsKey("pattern")) { + assert elem["pattern"] instanceof CharSequence + } + if (elem.containsKey("saveAs")) { + assert elem["saveAs"] instanceof CharSequence //: "saveAs as a Closure is currently not supported. Surround your closure with single quotes to get the desired effect. Example: '\{ foo \}'" + } + if (elem.containsKey("enabled")) { + assert elem["enabled"] instanceof Boolean + } + + // return final result + elem + } + // store final directive + drctv["publishDir"] = pblsh + } + + /* DIRECTIVE queue + accepted examples: + - "long" + - "short,long" + - ["short", "long"] + */ + if (drctv.containsKey("queue")) { + if (drctv["queue"] instanceof List) { + drctv["queue"] = drctv["queue"].join(",") + } + assert drctv["queue"] instanceof CharSequence + } + + /* DIRECTIVE label + accepted examples: + - "big_mem" + - "big_cpu" + - ["big_mem", "big_cpu"] + */ + if (drctv.containsKey("label")) { + if (drctv["label"] instanceof CharSequence) { + drctv["label"] = [ drctv["label"] ] + } + assert drctv["label"] instanceof List + drctv["label"].forEach { label -> + assert label instanceof CharSequence + // assert label.matches("[a-zA-Z0-9]([a-zA-Z0-9_]*[a-zA-Z0-9])?") + // ^ does not allow closures + } + } + + /* DIRECTIVE scratch + accepted examples: + - true + - "/path/to/scratch" + - '$MY_PATH_TO_SCRATCH' + - "ram-disk" + */ + if (drctv.containsKey("scratch")) { + assert drctv["scratch"] == true || drctv["scratch"] instanceof CharSequence + } + + /* DIRECTIVE storeDir + accepted examples: + - "/path/to/storeDir" + */ + if (drctv.containsKey("storeDir")) { + assert drctv["storeDir"] instanceof CharSequence + } + + /* DIRECTIVE stageInMode + accepted examples: + - "copy" + - "link" + */ + if (drctv.containsKey("stageInMode")) { + assert drctv["stageInMode"] instanceof CharSequence + assert drctv["stageInMode"] in ["copy", "link", "symlink", "rellink"] + } + + /* DIRECTIVE stageOutMode + accepted examples: + - "copy" + - "link" + */ + if (drctv.containsKey("stageOutMode")) { + assert drctv["stageOutMode"] instanceof CharSequence + assert drctv["stageOutMode"] in ["copy", "move", "rsync"] + } + + /* DIRECTIVE tag + accepted examples: + - "foo" + - '$id' + */ + if (drctv.containsKey("tag")) { + assert drctv["tag"] instanceof CharSequence + } + + /* DIRECTIVE time + accepted examples: + - "1h" + - "2days" + - "1day 6hours 3minutes 30seconds" + */ + if (drctv.containsKey("time")) { + assert drctv["time"] instanceof CharSequence + // todo: validation regex? + } + + return drctv +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/processWorkflowArgs.nf' +def processWorkflowArgs(Map args, Map defaultWfArgs, Map meta) { + // override defaults with args + def workflowArgs = defaultWfArgs + args + + // check whether 'key' exists + assert workflowArgs.containsKey("key") : "Error in module '${meta.config.name}': key is a required argument" + + // if 'key' is a closure, apply it to the original key + if (workflowArgs["key"] instanceof Closure) { + workflowArgs["key"] = workflowArgs["key"](meta.config.name) + } + def key = workflowArgs["key"] + assert key instanceof CharSequence : "Expected process argument 'key' to be a String. Found: class ${key.getClass()}" + assert key ==~ /^[a-zA-Z_]\w*$/ : "Error in module '$key': Expected process argument 'key' to consist of only letters, digits or underscores. Found: ${key}" + + // check for any unexpected keys + def expectedKeys = ["key", "directives", "auto", "map", "mapId", "mapData", "mapPassthrough", "filter", "runIf", "fromState", "toState", "args", "renameKeys", "debug"] + def unexpectedKeys = workflowArgs.keySet() - expectedKeys + assert unexpectedKeys.isEmpty() : "Error in module '$key': unexpected arguments to the '.run()' function: '${unexpectedKeys.join("', '")}'" + + // check whether directives exists and apply defaults + assert workflowArgs.containsKey("directives") : "Error in module '$key': directives is a required argument" + assert workflowArgs["directives"] instanceof Map : "Error in module '$key': Expected process argument 'directives' to be a Map. Found: class ${workflowArgs['directives'].getClass()}" + workflowArgs["directives"] = processDirectives(defaultWfArgs.directives + workflowArgs["directives"]) + + // check whether directives exists and apply defaults + assert workflowArgs.containsKey("auto") : "Error in module '$key': auto is a required argument" + assert workflowArgs["auto"] instanceof Map : "Error in module '$key': Expected process argument 'auto' to be a Map. Found: class ${workflowArgs['auto'].getClass()}" + workflowArgs["auto"] = processAuto(defaultWfArgs.auto + workflowArgs["auto"]) + + // auto define publish, if so desired + if (workflowArgs.auto.publish == true && (workflowArgs.directives.publishDir != null ? workflowArgs.directives.publishDir : [:]).isEmpty()) { + // can't assert at this level thanks to the no_publish profile + // assert params.containsKey("publishDir") || params.containsKey("publish_dir") : + // "Error in module '${workflowArgs['key']}': if auto.publish is true, params.publish_dir needs to be defined.\n" + + // " Example: params.publish_dir = \"./output/\"" + def publishDir = getPublishDir() + + if (publishDir != null) { + workflowArgs.directives.publishDir = [[ + path: publishDir, + saveAs: "{ it.startsWith('.') ? null : it }", // don't publish hidden files, by default + mode: "copy" + ]] + } + } + + // auto define transcript, if so desired + if (workflowArgs.auto.transcript == true) { + // can't assert at this level thanks to the no_publish profile + // assert params.containsKey("transcriptsDir") || params.containsKey("transcripts_dir") || params.containsKey("publishDir") || params.containsKey("publish_dir") : + // "Error in module '${workflowArgs['key']}': if auto.transcript is true, either params.transcripts_dir or params.publish_dir needs to be defined.\n" + + // " Example: params.transcripts_dir = \"./transcripts/\"" + def transcriptsDir = + params.containsKey("transcripts_dir") ? params.transcripts_dir : + params.containsKey("transcriptsDir") ? params.transcriptsDir : + params.containsKey("publish_dir") ? params.publish_dir + "/_transcripts" : + params.containsKey("publishDir") ? params.publishDir + "/_transcripts" : + null + if (transcriptsDir != null) { + def timestamp = nextflow.Nextflow.getSession().getWorkflowMetadata().start.format('yyyy-MM-dd_HH-mm-ss') + def transcriptsPublishDir = [ + path: "$transcriptsDir/$timestamp/\${task.process.replaceAll(':', '-')}/\${id}/", + saveAs: "{ it.startsWith('.') ? it.replaceAll('^.', '') : null }", + mode: "copy" + ] + def publishDirs = workflowArgs.directives.publishDir != null ? workflowArgs.directives.publishDir : null ? workflowArgs.directives.publishDir : [] + workflowArgs.directives.publishDir = publishDirs + transcriptsPublishDir + } + } + + // if this is a stubrun, remove certain directives? + if (workflow.stubRun) { + workflowArgs.directives.keySet().removeAll(["publishDir", "cpus", "memory", "label"]) + } + + for (nam in ["map", "mapId", "mapData", "mapPassthrough", "filter", "runIf"]) { + if (workflowArgs.containsKey(nam) && workflowArgs[nam]) { + assert workflowArgs[nam] instanceof Closure : "Error in module '$key': Expected process argument '$nam' to be null or a Closure. Found: class ${workflowArgs[nam].getClass()}" + } + } + + // TODO: should functions like 'map', 'mapId', 'mapData', 'mapPassthrough' be deprecated as well? + for (nam in ["map", "mapData", "mapPassthrough", "renameKeys"]) { + if (workflowArgs.containsKey(nam) && workflowArgs[nam] != null) { + log.warn "module '$key': workflow argument '$nam' is deprecated and will be removed in Viash 0.9.0. Please use 'fromState' and 'toState' instead." + } + } + + // check fromState + workflowArgs["fromState"] = _processFromState(workflowArgs.get("fromState"), key, meta.config) + + // check toState + workflowArgs["toState"] = _processToState(workflowArgs.get("toState"), key, meta.config) + + // return output + return workflowArgs +} + +def _processFromState(fromState, key_, config_) { + assert fromState == null || fromState instanceof Closure || fromState instanceof Map || fromState instanceof List : + "Error in module '$key_': Expected process argument 'fromState' to be null, a Closure, a Map, or a List. Found: class ${fromState.getClass()}" + if (fromState == null) { + return null + } + + // if fromState is a List, convert to map + if (fromState instanceof List) { + // check whether fromstate is a list[string] + assert fromState.every{it instanceof CharSequence} : "Error in module '$key_': fromState is a List, but not all elements are Strings" + fromState = fromState.collectEntries{[it, it]} + } + + // if fromState is a map, convert to closure + if (fromState instanceof Map) { + // check whether fromstate is a map[string, string] + assert fromState.values().every{it instanceof CharSequence} : "Error in module '$key_': fromState is a Map, but not all values are Strings" + assert fromState.keySet().every{it instanceof CharSequence} : "Error in module '$key_': fromState is a Map, but not all keys are Strings" + def fromStateMap = fromState.clone() + def requiredInputNames = meta.config.allArguments.findAll{it.required && it.direction == "Input"}.collect{it.plainName} + // turn the map into a closure to be used later on + fromState = { it -> + def state = it[1] + assert state instanceof Map : "Error in module '$key_': the state is not a Map" + def data = fromStateMap.collectMany{newkey, origkey -> + // check whether newkey corresponds to a required argument + if (state.containsKey(origkey)) { + [[newkey, state[origkey]]] + } else if (!requiredInputNames.contains(origkey)) { + [] + } else { + throw new Exception("Error in module '$key_': fromState key '$origkey' not found in current state") + } + }.collectEntries() + data + } + } + + return fromState +} + +def _processToState(toState, key_, config_) { + if (toState == null) { + toState = { tup -> tup[1] } + } + + // toState should be a closure, map[string, string], or list[string] + assert toState instanceof Closure || toState instanceof Map || toState instanceof List : + "Error in module '$key_': Expected process argument 'toState' to be a Closure, a Map, or a List. Found: class ${toState.getClass()}" + + // if toState is a List, convert to map + if (toState instanceof List) { + // check whether toState is a list[string] + assert toState.every{it instanceof CharSequence} : "Error in module '$key_': toState is a List, but not all elements are Strings" + toState = toState.collectEntries{[it, it]} + } + + // if toState is a map, convert to closure + if (toState instanceof Map) { + // check whether toState is a map[string, string] + assert toState.values().every{it instanceof CharSequence} : "Error in module '$key_': toState is a Map, but not all values are Strings" + assert toState.keySet().every{it instanceof CharSequence} : "Error in module '$key_': toState is a Map, but not all keys are Strings" + def toStateMap = toState.clone() + def requiredOutputNames = config_.allArguments.findAll{it.required && it.direction == "Output"}.collect{it.plainName} + // turn the map into a closure to be used later on + toState = { it -> + def output = it[1] + def state = it[2] + assert output instanceof Map : "Error in module '$key_': the output is not a Map" + assert state instanceof Map : "Error in module '$key_': the state is not a Map" + def extraEntries = toStateMap.collectMany{newkey, origkey -> + // check whether newkey corresponds to a required argument + if (output.containsKey(origkey)) { + [[newkey, output[origkey]]] + } else if (!requiredOutputNames.contains(origkey)) { + [] + } else { + throw new Exception("Error in module '$key_': toState key '$origkey' not found in current output") + } + }.collectEntries() + state + extraEntries + } + } + + return toState +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/workflowFactory.nf' +def _debug(workflowArgs, debugKey) { + if (workflowArgs.debug) { + view { "process '${workflowArgs.key}' $debugKey tuple: $it" } + } else { + map { it } + } +} + +// depends on: innerWorkflowFactory +def workflowFactory(Map args, Map defaultWfArgs, Map meta) { + def workflowArgs = processWorkflowArgs(args, defaultWfArgs, meta) + def key_ = workflowArgs["key"] + + workflow workflowInstance { + take: input_ + + main: + def chModified = input_ + | checkUniqueIds([:]) + | _debug(workflowArgs, "input") + | map { tuple -> + tuple = deepClone(tuple) + + if (workflowArgs.map) { + tuple = workflowArgs.map(tuple) + } + if (workflowArgs.mapId) { + tuple[0] = workflowArgs.mapId(tuple[0]) + } + if (workflowArgs.mapData) { + tuple[1] = workflowArgs.mapData(tuple[1]) + } + if (workflowArgs.mapPassthrough) { + tuple = tuple.take(2) + workflowArgs.mapPassthrough(tuple.drop(2)) + } + + // check tuple + assert tuple instanceof List : + "Error in module '${key_}': element in channel should be a tuple [id, data, ...otherargs...]\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Expected class: List. Found: tuple.getClass() is ${tuple.getClass()}" + assert tuple.size() >= 2 : + "Error in module '${key_}': expected length of tuple in input channel to be two or greater.\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Found: tuple.size() == ${tuple.size()}" + + // check id field + if (tuple[0] instanceof GString) { + tuple[0] = tuple[0].toString() + } + assert tuple[0] instanceof CharSequence : + "Error in module '${key_}': first element of tuple in channel should be a String\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Found: ${tuple[0]}" + + // match file to input file + if (workflowArgs.auto.simplifyInput && (tuple[1] instanceof Path || tuple[1] instanceof List)) { + def inputFiles = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "input" } + + assert inputFiles.size() == 1 : + "Error in module '${key_}' id '${tuple[0]}'.\n" + + " Anonymous file inputs are only allowed when the process has exactly one file input.\n" + + " Expected: inputFiles.size() == 1. Found: inputFiles.size() is ${inputFiles.size()}" + + tuple[1] = [[ inputFiles[0].plainName, tuple[1] ]].collectEntries() + } + + // check data field + assert tuple[1] instanceof Map : + "Error in module '${key_}' id '${tuple[0]}': second element of tuple in channel should be a Map\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Expected class: Map. Found: tuple[1].getClass() is ${tuple[1].getClass()}" + + // rename keys of data field in tuple + if (workflowArgs.renameKeys) { + assert workflowArgs.renameKeys instanceof Map : + "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + + " Example: renameKeys: ['new_key': 'old_key'].\n" + + " Expected class: Map. Found: renameKeys.getClass() is ${workflowArgs.renameKeys.getClass()}" + assert tuple[1] instanceof Map : + "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + + " Expected class: Map. Found: tuple[1].getClass() is ${tuple[1].getClass()}" + + // TODO: allow renameKeys to be a function? + workflowArgs.renameKeys.each { newKey, oldKey -> + assert newKey instanceof CharSequence : + "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + + " Example: renameKeys: ['new_key': 'old_key'].\n" + + " Expected class of newKey: String. Found: newKey.getClass() is ${newKey.getClass()}" + assert oldKey instanceof CharSequence : + "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + + " Example: renameKeys: ['new_key': 'old_key'].\n" + + " Expected class of oldKey: String. Found: oldKey.getClass() is ${oldKey.getClass()}" + assert tuple[1].containsKey(oldKey) : + "Error renaming data keys in module '${key}' id '${tuple[0]}'.\n" + + " Key '$oldKey' is missing in the data map. tuple[1].keySet() is '${tuple[1].keySet()}'" + tuple[1].put(newKey, tuple[1][oldKey]) + } + tuple[1].keySet().removeAll(workflowArgs.renameKeys.collect{ newKey, oldKey -> oldKey }) + } + tuple + } + + + def chRun = null + def chPassthrough = null + if (workflowArgs.runIf) { + def runIfBranch = chModified.branch{ tup -> + run: workflowArgs.runIf(tup[0], tup[1]) + passthrough: true + } + chRun = runIfBranch.run + chPassthrough = runIfBranch.passthrough + } else { + chRun = chModified + chPassthrough = Channel.empty() + } + + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + + def chArgs = workflowArgs.fromState ? + chRunFiltered | map{ + def new_data = workflowArgs.fromState(it.take(2)) + [it[0], new_data] + } : + chRunFiltered | map {tup -> tup.take(2)} + + // fill in defaults + def chArgsWithDefaults = chArgs + | map { tuple -> + def id_ = tuple[0] + def data_ = tuple[1] + + // TODO: could move fromState to here + + // fetch default params from functionality + def defaultArgs = meta.config.allArguments + .findAll { it.containsKey("default") } + .collectEntries { [ it.plainName, it.default ] } + + // fetch overrides in params + def paramArgs = meta.config.allArguments + .findAll { par -> + def argKey = key_ + "__" + par.plainName + params.containsKey(argKey) + } + .collectEntries { [ it.plainName, params[key_ + "__" + it.plainName] ] } + + // fetch overrides in data + def dataArgs = meta.config.allArguments + .findAll { data_.containsKey(it.plainName) } + .collectEntries { [ it.plainName, data_[it.plainName] ] } + + // combine params + def combinedArgs = defaultArgs + paramArgs + workflowArgs.args + dataArgs + + // remove arguments with explicit null values + combinedArgs + .removeAll{_, val -> val == null || val == "viash_no_value" || val == "force_null"} + + combinedArgs = _processInputValues(combinedArgs, meta.config, id_, key_) + + [id_, combinedArgs] + tuple.drop(2) + } + + // TODO: move some of the _meta.join_id wrangling to the safeJoin() function. + def chInitialOutput = chArgsWithDefaults + | _debug(workflowArgs, "processed") + // run workflow + | innerWorkflowFactory(workflowArgs) + // check output tuple + | map { id_, output_ -> + + // see if output map contains metadata + def meta_ = + output_ instanceof Map && output_.containsKey("_meta") ? + output_["_meta"] : + [:] + def join_id = meta_.join_id ?: id_ + + // remove metadata + output_ = output_.findAll{k, v -> k != "_meta"} + + // check value types + output_ = _processOutputValues(output_, meta.config, id_, key_) + + // simplify output if need be + if (workflowArgs.auto.simplifyOutput && output_.size() == 1) { + output_ = output_.values()[0] + } + + [join_id, id_, output_] + } + // | view{"chInitialOutput: ${it.take(3)}"} + + // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) + // input tuple format: [join_id, id, output, prev_state, ...] + // output tuple format: [join_id, id, new_state, ...] + | map{ tup -> + def new_state = workflowArgs.toState(tup.drop(1).take(3)) + tup.take(2) + [new_state] + tup.drop(4) + } + + if (workflowArgs.auto.publish == "state") { + def chPublish = chNewState + // input tuple format: [join_id, id, new_state, ...] + // output tuple format: [join_id, id, new_state] + | map{ tup -> + tup.take(3) + } + + safeJoin(chPublish, chArgsWithDefaults, key_) + // input tuple format: [join_id, id, new_state, orig_state, ...] + // output tuple format: [id, new_state, orig_state] + | map { tup -> + tup.drop(1).take(3) + } + | publishStatesByConfig(key: key_, config: meta.config) + } + + // remove join_id and meta + chReturn = chNewState + | map { tup -> + // input tuple format: [join_id, id, new_state, ...] + // output tuple format: [id, new_state, ...] + tup.drop(1) + } + | _debug(workflowArgs, "output") + | concat(chPassthrough) + + emit: chReturn + } + + def wf = workflowInstance.cloneWithName(key_) + + // add factory function + wf.metaClass.run = { runArgs -> + workflowFactory(runArgs, workflowArgs, meta) + } + // add config to module for later introspection + wf.metaClass.config = meta.config + + return wf +} + +nextflow.enable.dsl=2 + +// START COMPONENT-SPECIFIC CODE + +// create meta object +meta = [ + "resources_dir": moduleDir.toRealPath().normalize(), + "config": processConfig(readJsonBlob('''{ + "name" : "bedtools_merge", + "namespace" : "bedtools", + "version" : "main", + "authors" : [ + { + "name" : "Theodoro Gasperin Terra Camargo", + "roles" : [ + "author", + "maintainer" + ], + "info" : { + "links" : { + "email" : "theodorogtc@gmail.com", + "github" : "tgaspe", + "linkedin" : "theodoro-gasperin-terra-camargo" + }, + "organizations" : [ + { + "name" : "Data Intuitive", + "href" : "https://www.data-intuitive.com", + "role" : "Bioinformatician" + } + ] + } + } + ], + "argument_groups" : [ + { + "name" : "Inputs", + "arguments" : [ + { + "type" : "file", + "name" : "--input", + "alternatives" : [ + "-i" + ], + "description" : "Input file (BED/GFF/VCF) to be merged.", + "must_exist" : true, + "create_parent" : true, + "required" : true, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + } + ] + }, + { + "name" : "Outputs", + "arguments" : [ + { + "type" : "file", + "name" : "--output", + "description" : "Output merged file BED to be written.", + "must_exist" : true, + "create_parent" : true, + "required" : true, + "direction" : "output", + "multiple" : false, + "multiple_sep" : ";" + } + ] + }, + { + "name" : "Options", + "arguments" : [ + { + "type" : "boolean_true", + "name" : "--strand", + "alternatives" : [ + "-s" + ], + "description" : "Force strandedness. That is, only merge features\nthat are on the same strand.\n- By default, merging is done without respect to strand.\n", + "direction" : "input" + }, + { + "type" : "string", + "name" : "--specific_strand", + "alternatives" : [ + "-S" + ], + "description" : "Force merge for one specific strand only.\nFollow with + or - to force merge from only\nthe forward or reverse strand, respectively.\n- By default, merging is done without respect to strand.\n", + "required" : false, + "choices" : [ + "+", + "-" + ], + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + }, + { + "type" : "integer", + "name" : "--distance", + "alternatives" : [ + "-d" + ], + "description" : "Maximum distance between features allowed for features\nto be merged.\n- Def. 0. That is, overlapping & book-ended features are merged.\n- (INTEGER)\n- Note: negative values enforce the number of b.p. required for overlap.\n", + "required" : false, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + }, + { + "type" : "integer", + "name" : "--columns", + "alternatives" : [ + "-c" + ], + "description" : "Specify columns from the B file to map onto intervals in A.\nDefault: 5.\nMultiple columns can be specified in a comma-delimited list.\n", + "required" : false, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + }, + { + "type" : "string", + "name" : "--operation", + "alternatives" : [ + "-o" + ], + "description" : "Specify the operation that should be applied to -c.\nValid operations:\n sum, min, max, absmin, absmax,\n mean, median, mode, antimode\n stdev, sstdev\n collapse (i.e., print a delimited list (duplicates allowed)), \n distinct (i.e., print a delimited list (NO duplicates allowed)), \n distinct_sort_num (as distinct, sorted numerically, ascending),\n distinct_sort_num_desc (as distinct, sorted numerically, desscending),\n distinct_only (delimited list of only unique values),\n count\n count_distinct (i.e., a count of the unique values in the column), \n first (i.e., just the first value in the column), \n last (i.e., just the last value in the column), \nDefault: sum\nMultiple operations can be specified in a comma-delimited list.\n\nIf there is only column, but multiple operations, all operations will be\napplied on that column. Likewise, if there is only one operation, but\nmultiple columns, that operation will be applied to all columns.\nOtherwise, the number of columns must match the the number of operations,\nand will be applied in respective order.\nE.g., \\"-c 5,4,6 -o sum,mean,count\\" will give the sum of column 5,\nthe mean of column 4, and the count of column 6.\nThe order of output columns will match the ordering given in the command.\n", + "required" : false, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + }, + { + "type" : "string", + "name" : "--delimiter", + "alternatives" : [ + "-delim" + ], + "description" : "Specify a custom delimiter for the collapse operations.\n", + "example" : [ + "|" + ], + "default" : [ + "," + ], + "required" : false, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + }, + { + "type" : "integer", + "name" : "--precision", + "alternatives" : [ + "-prec" + ], + "description" : "Sets the decimal precision for output (Default: 5).\n", + "required" : false, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + }, + { + "type" : "boolean_true", + "name" : "--bed", + "description" : "If using BAM input, write output as BED.\n", + "direction" : "input" + }, + { + "type" : "boolean_true", + "name" : "--header", + "description" : "Print the header from the A file prior to results.\n", + "direction" : "input" + }, + { + "type" : "boolean_true", + "name" : "--no_buffer", + "alternatives" : [ + "-nobuf" + ], + "description" : "Disable buffered output. Using this option will cause each line\nof output to be printed as it is generated, rather than saved\nin a buffer. This will make printing large output files \nnoticeably slower, but can be useful in conjunction with\nother software tools and scripts that need to process one\nline of bedtools output at a time.\n", + "direction" : "input" + } + ] + } + ], + "resources" : [ + { + "type" : "bash_script", + "path" : "script.sh", + "is_executable" : true + } + ], + "description" : "Merges overlapping BED/GFF/VCF entries into a single interval.\n", + "test_resources" : [ + { + "type" : "bash_script", + "path" : "test.sh", + "is_executable" : true + }, + { + "type" : "file", + "path" : "test_data" + } + ], + "status" : "enabled", + "requirements" : { + "commands" : [ + "ps" + ] + }, + "license" : "MIT", + "references" : { + "doi" : [ + "10.1093/bioinformatics/btq033" + ] + }, + "links" : { + "repository" : "https://github.com/arq5x/bedtools2", + "homepage" : "https://bedtools.readthedocs.io/en/latest/#", + "documentation" : "https://bedtools.readthedocs.io/en/latest/content/tools/merge.html", + "issue_tracker" : "https://github.com/arq5x/bedtools2/issues" + }, + "runners" : [ + { + "type" : "executable", + "id" : "executable", + "docker_setup_strategy" : "ifneedbepullelsecachedbuild" + }, + { + "type" : "nextflow", + "id" : "nextflow", + "directives" : { + "tag" : "$id" + }, + "auto" : { + "simplifyInput" : true, + "simplifyOutput" : false, + "transcript" : false, + "publish" : false + }, + "config" : { + "labels" : { + "mem1gb" : "memory = 1000000000.B", + "mem2gb" : "memory = 2000000000.B", + "mem5gb" : "memory = 5000000000.B", + "mem10gb" : "memory = 10000000000.B", + "mem20gb" : "memory = 20000000000.B", + "mem50gb" : "memory = 50000000000.B", + "mem100gb" : "memory = 100000000000.B", + "mem200gb" : "memory = 200000000000.B", + "mem500gb" : "memory = 500000000000.B", + "mem1tb" : "memory = 1000000000000.B", + "mem2tb" : "memory = 2000000000000.B", + "mem5tb" : "memory = 5000000000000.B", + "mem10tb" : "memory = 10000000000000.B", + "mem20tb" : "memory = 20000000000000.B", + "mem50tb" : "memory = 50000000000000.B", + "mem100tb" : "memory = 100000000000000.B", + "mem200tb" : "memory = 200000000000000.B", + "mem500tb" : "memory = 500000000000000.B", + "mem1gib" : "memory = 1073741824.B", + "mem2gib" : "memory = 2147483648.B", + "mem4gib" : "memory = 4294967296.B", + "mem8gib" : "memory = 8589934592.B", + "mem16gib" : "memory = 17179869184.B", + "mem32gib" : "memory = 34359738368.B", + "mem64gib" : "memory = 68719476736.B", + "mem128gib" : "memory = 137438953472.B", + "mem256gib" : "memory = 274877906944.B", + "mem512gib" : "memory = 549755813888.B", + "mem1tib" : "memory = 1099511627776.B", + "mem2tib" : "memory = 2199023255552.B", + "mem4tib" : "memory = 4398046511104.B", + "mem8tib" : "memory = 8796093022208.B", + "mem16tib" : "memory = 17592186044416.B", + "mem32tib" : "memory = 35184372088832.B", + "mem64tib" : "memory = 70368744177664.B", + "mem128tib" : "memory = 140737488355328.B", + "mem256tib" : "memory = 281474976710656.B", + "mem512tib" : "memory = 562949953421312.B", + "cpu1" : "cpus = 1", + "cpu2" : "cpus = 2", + "cpu5" : "cpus = 5", + "cpu10" : "cpus = 10", + "cpu20" : "cpus = 20", + "cpu50" : "cpus = 50", + "cpu100" : "cpus = 100", + "cpu200" : "cpus = 200", + "cpu500" : "cpus = 500", + "cpu1000" : "cpus = 1000" + } + }, + "debug" : false, + "container" : "docker" + } + ], + "engines" : [ + { + "type" : "docker", + "id" : "docker", + "image" : "debian:stable-slim", + "target_registry" : "images.viash-hub.com", + "target_tag" : "main", + "namespace_separator" : "/", + "setup" : [ + { + "type" : "apt", + "packages" : [ + "bedtools", + "procps" + ], + "interactive" : false + }, + { + "type" : "docker", + "run" : [ + "echo \\"bedtools: \\\\\\"$(bedtools --version | sed -n 's/^bedtools //p')\\\\\\"\\" > /var/software_versions.txt\n" + ] + } + ] + }, + { + "type" : "native", + "id" : "native" + } + ], + "build_info" : { + "config" : "/workdir/root/repo/src/bedtools/bedtools_merge/config.vsh.yaml", + "runner" : "nextflow", + "engine" : "docker|native", + "output" : "target/nextflow/bedtools/bedtools_merge", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", + "git_remote" : "https://github.com/viash-hub/biobox" + }, + "package_config" : { + "name" : "biobox", + "version" : "main", + "description" : "A collection of bioinformatics tools for working with sequence data.\n", + "viash_version" : "0.9.0-RC7", + "source" : "src", + "target" : "target", + "config_mods" : [ + ".requirements.commands := ['ps']\n", + ".engines += { type: \\"native\\" }", + ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", + ".engines[.type == 'docker'].target_tag := 'main'" + ], + "keywords" : [ + "bioinformatics", + "modules", + "sequencing" + ], + "license" : "MIT", + "organization" : "vsh", + "links" : { + "repository" : "https://github.com/viash-hub/biobox", + "issue_tracker" : "https://github.com/viash-hub/biobox/issues" + } + } +}''')) +] + +// resolve dependencies dependencies (if any) + + +// inner workflow +// inner workflow hook +def innerWorkflowFactory(args) { + def rawScript = '''set -e +tempscript=".viash_script.sh" +cat > "$tempscript" << VIASHMAIN +#!/bin/bash + +## VIASH START +# The following code has been auto-generated by Viash. +$( if [ ! -z ${VIASH_PAR_INPUT+x} ]; then echo "${VIASH_PAR_INPUT}" | sed "s#'#'\\"'\\"'#g;s#.*#par_input='&'#" ; else echo "# par_input="; fi ) +$( if [ ! -z ${VIASH_PAR_OUTPUT+x} ]; then echo "${VIASH_PAR_OUTPUT}" | sed "s#'#'\\"'\\"'#g;s#.*#par_output='&'#" ; else echo "# par_output="; fi ) +$( if [ ! -z ${VIASH_PAR_STRAND+x} ]; then echo "${VIASH_PAR_STRAND}" | sed "s#'#'\\"'\\"'#g;s#.*#par_strand='&'#" ; else echo "# par_strand="; fi ) +$( if [ ! -z ${VIASH_PAR_SPECIFIC_STRAND+x} ]; then echo "${VIASH_PAR_SPECIFIC_STRAND}" | sed "s#'#'\\"'\\"'#g;s#.*#par_specific_strand='&'#" ; else echo "# par_specific_strand="; fi ) +$( if [ ! -z ${VIASH_PAR_DISTANCE+x} ]; then echo "${VIASH_PAR_DISTANCE}" | sed "s#'#'\\"'\\"'#g;s#.*#par_distance='&'#" ; else echo "# par_distance="; fi ) +$( if [ ! -z ${VIASH_PAR_COLUMNS+x} ]; then echo "${VIASH_PAR_COLUMNS}" | sed "s#'#'\\"'\\"'#g;s#.*#par_columns='&'#" ; else echo "# par_columns="; fi ) +$( if [ ! -z ${VIASH_PAR_OPERATION+x} ]; then echo "${VIASH_PAR_OPERATION}" | sed "s#'#'\\"'\\"'#g;s#.*#par_operation='&'#" ; else echo "# par_operation="; fi ) +$( if [ ! -z ${VIASH_PAR_DELIMITER+x} ]; then echo "${VIASH_PAR_DELIMITER}" | sed "s#'#'\\"'\\"'#g;s#.*#par_delimiter='&'#" ; else echo "# par_delimiter="; fi ) +$( if [ ! -z ${VIASH_PAR_PRECISION+x} ]; then echo "${VIASH_PAR_PRECISION}" | sed "s#'#'\\"'\\"'#g;s#.*#par_precision='&'#" ; else echo "# par_precision="; fi ) +$( if [ ! -z ${VIASH_PAR_BED+x} ]; then echo "${VIASH_PAR_BED}" | sed "s#'#'\\"'\\"'#g;s#.*#par_bed='&'#" ; else echo "# par_bed="; fi ) +$( if [ ! -z ${VIASH_PAR_HEADER+x} ]; then echo "${VIASH_PAR_HEADER}" | sed "s#'#'\\"'\\"'#g;s#.*#par_header='&'#" ; else echo "# par_header="; fi ) +$( if [ ! -z ${VIASH_PAR_NO_BUFFER+x} ]; then echo "${VIASH_PAR_NO_BUFFER}" | sed "s#'#'\\"'\\"'#g;s#.*#par_no_buffer='&'#" ; else echo "# par_no_buffer="; fi ) +$( if [ ! -z ${VIASH_META_NAME+x} ]; then echo "${VIASH_META_NAME}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_name='&'#" ; else echo "# meta_name="; fi ) +$( if [ ! -z ${VIASH_META_FUNCTIONALITY_NAME+x} ]; then echo "${VIASH_META_FUNCTIONALITY_NAME}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_functionality_name='&'#" ; else echo "# meta_functionality_name="; fi ) +$( if [ ! -z ${VIASH_META_RESOURCES_DIR+x} ]; then echo "${VIASH_META_RESOURCES_DIR}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_resources_dir='&'#" ; else echo "# meta_resources_dir="; fi ) +$( if [ ! -z ${VIASH_META_EXECUTABLE+x} ]; then echo "${VIASH_META_EXECUTABLE}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_executable='&'#" ; else echo "# meta_executable="; fi ) +$( if [ ! -z ${VIASH_META_CONFIG+x} ]; then echo "${VIASH_META_CONFIG}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_config='&'#" ; else echo "# meta_config="; fi ) +$( if [ ! -z ${VIASH_META_TEMP_DIR+x} ]; then echo "${VIASH_META_TEMP_DIR}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_temp_dir='&'#" ; else echo "# meta_temp_dir="; fi ) +$( if [ ! -z ${VIASH_META_CPUS+x} ]; then echo "${VIASH_META_CPUS}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_cpus='&'#" ; else echo "# meta_cpus="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_B+x} ]; then echo "${VIASH_META_MEMORY_B}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_b='&'#" ; else echo "# meta_memory_b="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_KB+x} ]; then echo "${VIASH_META_MEMORY_KB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_kb='&'#" ; else echo "# meta_memory_kb="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_MB+x} ]; then echo "${VIASH_META_MEMORY_MB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_mb='&'#" ; else echo "# meta_memory_mb="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_GB+x} ]; then echo "${VIASH_META_MEMORY_GB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_gb='&'#" ; else echo "# meta_memory_gb="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_TB+x} ]; then echo "${VIASH_META_MEMORY_TB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_tb='&'#" ; else echo "# meta_memory_tb="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_PB+x} ]; then echo "${VIASH_META_MEMORY_PB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_pb='&'#" ; else echo "# meta_memory_pb="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_KIB+x} ]; then echo "${VIASH_META_MEMORY_KIB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_kib='&'#" ; else echo "# meta_memory_kib="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_MIB+x} ]; then echo "${VIASH_META_MEMORY_MIB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_mib='&'#" ; else echo "# meta_memory_mib="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_GIB+x} ]; then echo "${VIASH_META_MEMORY_GIB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_gib='&'#" ; else echo "# meta_memory_gib="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_TIB+x} ]; then echo "${VIASH_META_MEMORY_TIB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_tib='&'#" ; else echo "# meta_memory_tib="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_PIB+x} ]; then echo "${VIASH_META_MEMORY_PIB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_pib='&'#" ; else echo "# meta_memory_pib="; fi ) + +## VIASH END + +# Exit on error +set -eo pipefail + +# Unset parameters +unset_if_false=( + par_strand + par_bed + par_header + par_no_buffer +) + +for par in \\${unset_if_false[@]}; do + test_val="\\${!par}" + [[ "\\$test_val" == "false" ]] && unset \\$par +done + +# Execute bedtools merge with the provided arguments +bedtools merge \\\\ + \\${par_strand:+-s} \\\\ + \\${par_specific_strand:+-S "\\$par_specific_strand"} \\\\ + \\${par_bed:+-bed} \\\\ + \\${par_header:+-header} \\\\ + \\${par_no_buffer:+-nobuf} \\\\ + \\${par_distance:+-d "\\$par_distance"} \\\\ + \\${par_columns:+-c "\\$par_columns"} \\\\ + \\${par_operation:+-o "\\$par_operation"} \\\\ + \\${par_delimiter:+-delim "\\$par_delimiter"} \\\\ + \\${par_precision:+-prec "\\$par_precision"} \\\\ + -i "\\$par_input" \\\\ + > "\\$par_output" +VIASHMAIN +bash "$tempscript" +''' + + return vdsl3WorkflowFactory(args, meta, rawScript) +} + + + +/** + * Generate a workflow for VDSL3 modules. + * + * This function is called by the workflowFactory() function. + * + * Input channel: [id, input_map] + * Output channel: [id, output_map] + * + * Internally, this workflow will convert the input channel + * to a format which the Nextflow module will be able to handle. + */ +def vdsl3WorkflowFactory(Map args, Map meta, String rawScript) { + def key = args["key"] + def processObj = null + + workflow processWf { + take: input_ + main: + + if (processObj == null) { + processObj = _vdsl3ProcessFactory(args, meta, rawScript) + } + + output_ = input_ + | map { tuple -> + def id = tuple[0] + def data_ = tuple[1] + + if (workflow.stubRun) { + // add id if missing + data_ = [id: 'stub'] + data_ + } + + // process input files separately + def inputPaths = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "input" } + .collect { par -> + def val = data_.containsKey(par.plainName) ? data_[par.plainName] : [] + def inputFiles = [] + if (val == null) { + inputFiles = [] + } else if (val instanceof List) { + inputFiles = val + } else if (val instanceof Path) { + inputFiles = [ val ] + } else { + inputFiles = [] + } + if (!workflow.stubRun) { + // throw error when an input file doesn't exist + inputFiles.each{ file -> + assert file.exists() : + "Error in module '${key}' id '${id}' argument '${par.plainName}'.\n" + + " Required input file does not exist.\n" + + " Path: '$file'.\n" + + " Expected input file to exist" + } + } + inputFiles + } + + // remove input files + def argsExclInputFiles = meta.config.allArguments + .findAll { (it.type != "file" || it.direction != "input") && data_.containsKey(it.plainName) } + .collectEntries { par -> + def parName = par.plainName + def val = data_[parName] + if (par.multiple && val instanceof Collection) { + val = val.join(par.multiple_sep) + } + if (par.direction == "output" && par.type == "file") { + val = val.replaceAll('\\$id', id).replaceAll('\\$key', key) + } + [parName, val] + } + + [ id ] + inputPaths + [ argsExclInputFiles, meta.resources_dir ] + } + | processObj + | map { output -> + def outputFiles = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "output" } + .indexed() + .collectEntries{ index, par -> + def out = output[index + 1] + // strip dummy '.exitcode' file from output (see nextflow-io/nextflow#2678) + if (!out instanceof List || out.size() <= 1) { + if (par.multiple) { + out = [] + } else { + assert !par.required : + "Error in module '${key}' id '${output[0]}' argument '${par.plainName}'.\n" + + " Required output file is missing" + out = null + } + } else if (out.size() == 2 && !par.multiple) { + out = out[1] + } else { + out = out.drop(1) + } + [ par.plainName, out ] + } + + // drop null outputs + outputFiles.removeAll{it.value == null} + + [ output[0], outputFiles ] + } + emit: output_ + } + + return processWf +} + +// depends on: session? +def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) { + // autodetect process key + def wfKey = workflowArgs["key"] + def procKeyPrefix = "${wfKey}_process" + def scriptMeta = nextflow.script.ScriptMeta.current() + def existing = scriptMeta.getProcessNames().findAll{it.startsWith(procKeyPrefix)} + def numbers = existing.collect{it.replace(procKeyPrefix, "0").toInteger()} + def newNumber = (numbers + [-1]).max() + 1 + + def procKey = newNumber == 0 ? procKeyPrefix : "$procKeyPrefix$newNumber" + + if (newNumber > 0) { + log.warn "Key for module '${wfKey}' is duplicated.\n", + "If you run a component multiple times in the same workflow,\n" + + "it's recommended you set a unique key for every call,\n" + + "for example: ${wfKey}.run(key: \"foo\")." + } + + // subset directives and convert to list of tuples + def drctv = workflowArgs.directives + + // TODO: unit test the two commands below + // convert publish array into tags + def valueToStr = { val -> + // ignore closures + if (val instanceof CharSequence) { + if (!val.matches('^[{].*[}]$')) { + '"' + val + '"' + } else { + val + } + } else if (val instanceof List) { + "[" + val.collect{valueToStr(it)}.join(", ") + "]" + } else if (val instanceof Map) { + "[" + val.collect{k, v -> k + ": " + valueToStr(v)}.join(", ") + "]" + } else { + val.inspect() + } + } + + // multiple entries allowed: label, publishdir + def drctvStrs = drctv.collect { key, value -> + if (key in ["label", "publishDir"]) { + value.collect{ val -> + if (val instanceof Map) { + "\n$key " + val.collect{ k, v -> k + ": " + valueToStr(v) }.join(", ") + } else if (val == null) { + "" + } else { + "\n$key " + valueToStr(val) + } + }.join() + } else if (value instanceof Map) { + "\n$key " + value.collect{ k, v -> k + ": " + valueToStr(v) }.join(", ") + } else { + "\n$key " + valueToStr(value) + } + }.join() + + def inputPaths = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "input" } + .collect { ', path(viash_par_' + it.plainName + ', stageAs: "_viash_par/' + it.plainName + '_?/*")' } + .join() + + def outputPaths = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "output" } + .collect { par -> + // insert dummy into every output (see nextflow-io/nextflow#2678) + if (!par.multiple) { + ', path{[".exitcode", args.' + par.plainName + ']}' + } else { + ', path{[".exitcode"] + args.' + par.plainName + '}' + } + } + .join() + + // TODO: move this functionality somewhere else? + if (workflowArgs.auto.transcript) { + outputPaths = outputPaths + ', path{[".exitcode", ".command*"]}' + } else { + outputPaths = outputPaths + ', path{[".exitcode"]}' + } + + // create dirs for output files (based on BashWrapper.createParentFiles) + def createParentStr = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "output" && it.create_parent } + .collect { par -> + "\${ args.containsKey(\"${par.plainName}\") ? \"mkdir_parent \\\"\" + (args[\"${par.plainName}\"] instanceof String ? args[\"${par.plainName}\"] : args[\"${par.plainName}\"].join('\" \"')) + \"\\\"\" : \"\" }" + } + .join("\n") + + // construct inputFileExports + def inputFileExports = meta.config.allArguments + .findAll { it.type == "file" && it.direction.toLowerCase() == "input" } + .collect { par -> + def viash_par_contents = "(viash_par_${par.plainName} instanceof List ? viash_par_${par.plainName}.join(\"${par.multiple_sep}\") : viash_par_${par.plainName})" + "\n\${viash_par_${par.plainName}.empty ? \"\" : \"export VIASH_PAR_${par.plainName.toUpperCase()}=\\\"\" + ${viash_par_contents} + \"\\\"\"}" + } + + // NOTE: if using docker, use /tmp instead of tmpDir! + def tmpDir = java.nio.file.Paths.get( + System.getenv('NXF_TEMP') ?: + System.getenv('VIASH_TEMP') ?: + System.getenv('VIASH_TMPDIR') ?: + System.getenv('VIASH_TEMPDIR') ?: + System.getenv('VIASH_TMP') ?: + System.getenv('TEMP') ?: + System.getenv('TMPDIR') ?: + System.getenv('TEMPDIR') ?: + System.getenv('TMP') ?: + '/tmp' + ).toAbsolutePath() + + // construct stub + def stub = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "output" } + .collect { par -> + "\${ args.containsKey(\"${par.plainName}\") ? \"touch2 \\\"\" + (args[\"${par.plainName}\"] instanceof String ? args[\"${par.plainName}\"].replace(\"_*\", \"_0\") : args[\"${par.plainName}\"].join('\" \"')) + \"\\\"\" : \"\" }" + } + .join("\n") + + // escape script + def escapedScript = rawScript.replace('\\', '\\\\').replace('$', '\\$').replace('"""', '\\"\\"\\"') + + // publishdir assert + def assertStr = (workflowArgs.auto.publish == true) || workflowArgs.auto.transcript ? + """\nassert task.publishDir.size() > 0: "if auto.publish is true, params.publish_dir needs to be defined.\\n Example: --publish_dir './output/'" """ : + "" + + // generate process string + def procStr = + """nextflow.enable.dsl=2 + | + |process $procKey {$drctvStrs + |input: + | tuple val(id)$inputPaths, val(args), path(resourcesDir, stageAs: ".viash_meta_resources") + |output: + | tuple val("\$id")$outputPaths, optional: true + |stub: + |\"\"\" + |touch2() { mkdir -p "\\\$(dirname "\\\$1")" && touch "\\\$1" ; } + |$stub + |\"\"\" + |script:$assertStr + |def escapeText = { s -> s.toString().replaceAll('([`"])', '\\\\\\\\\$1') } + |def parInject = args + | .findAll{key, value -> value != null} + | .collect{key, value -> "export VIASH_PAR_\${key.toUpperCase()}=\\\"\${escapeText(value)}\\\""} + | .join("\\n") + |\"\"\" + |# meta exports + |export VIASH_META_RESOURCES_DIR="\${resourcesDir}" + |export VIASH_META_TEMP_DIR="${['docker', 'podman', 'charliecloud'].any{ it == workflow.containerEngine } ? '/tmp' : tmpDir}" + |export VIASH_META_NAME="${meta.config.name}" + |# export VIASH_META_EXECUTABLE="\\\$VIASH_META_RESOURCES_DIR/\\\$VIASH_META_NAME" + |export VIASH_META_CONFIG="\\\$VIASH_META_RESOURCES_DIR/.config.vsh.yaml" + |\${task.cpus ? "export VIASH_META_CPUS=\$task.cpus" : "" } + |\${task.memory?.bytes != null ? "export VIASH_META_MEMORY_B=\$task.memory.bytes" : "" } + |if [ ! -z \\\${VIASH_META_MEMORY_B+x} ]; then + | export VIASH_META_MEMORY_KB=\\\$(( (\\\$VIASH_META_MEMORY_B+999) / 1000 )) + | export VIASH_META_MEMORY_MB=\\\$(( (\\\$VIASH_META_MEMORY_KB+999) / 1000 )) + | export VIASH_META_MEMORY_GB=\\\$(( (\\\$VIASH_META_MEMORY_MB+999) / 1000 )) + | export VIASH_META_MEMORY_TB=\\\$(( (\\\$VIASH_META_MEMORY_GB+999) / 1000 )) + | export VIASH_META_MEMORY_PB=\\\$(( (\\\$VIASH_META_MEMORY_TB+999) / 1000 )) + | export VIASH_META_MEMORY_KIB=\\\$(( (\\\$VIASH_META_MEMORY_B+1023) / 1024 )) + | export VIASH_META_MEMORY_MIB=\\\$(( (\\\$VIASH_META_MEMORY_KIB+1023) / 1024 )) + | export VIASH_META_MEMORY_GIB=\\\$(( (\\\$VIASH_META_MEMORY_MIB+1023) / 1024 )) + | export VIASH_META_MEMORY_TIB=\\\$(( (\\\$VIASH_META_MEMORY_GIB+1023) / 1024 )) + | export VIASH_META_MEMORY_PIB=\\\$(( (\\\$VIASH_META_MEMORY_TIB+1023) / 1024 )) + |fi + | + |# meta synonyms + |export VIASH_TEMP="\\\$VIASH_META_TEMP_DIR" + |export TEMP_DIR="\\\$VIASH_META_TEMP_DIR" + | + |# create output dirs if need be + |function mkdir_parent { + | for file in "\\\$@"; do + | mkdir -p "\\\$(dirname "\\\$file")" + | done + |} + |$createParentStr + | + |# argument exports${inputFileExports.join()} + |\$parInject + | + |# process script + |${escapedScript} + |\"\"\" + |} + |""".stripMargin() + + // TODO: print on debug + // if (workflowArgs.debug == true) { + // println("######################\n$procStr\n######################") + // } + + // write process to temp file + def tempFile = java.nio.file.Files.createTempFile("viash-process-${procKey}-", ".nf") + addShutdownHook { java.nio.file.Files.deleteIfExists(tempFile) } + tempFile.text = procStr + + // create process from temp file + def binding = new nextflow.script.ScriptBinding([:]) + def session = nextflow.Nextflow.getSession() + def parser = new nextflow.script.ScriptParser(session) + .setModule(true) + .setBinding(binding) + def moduleScript = parser.runScript(tempFile) + .getScript() + + // register module in meta + def module = new nextflow.script.IncludeDef.Module(name: procKey) + scriptMeta.addModule(moduleScript, module.name, module.alias) + + // retrieve and return process from meta + return scriptMeta.getProcess(procKey) +} + +// defaults +meta["defaults"] = [ + // key to be used to trace the process and determine output names + key: null, + + // fixed arguments to be passed to script + args: [:], + + // default directives + directives: readJsonBlob('''{ + "container" : { + "registry" : "images.viash-hub.com", + "image" : "vsh/biobox/bedtools/bedtools_merge", + "tag" : "main" + }, + "tag" : "$id" +}'''), + + // auto settings + auto: readJsonBlob('''{ + "simplifyInput" : true, + "simplifyOutput" : false, + "transcript" : false, + "publish" : false +}'''), + + // Apply a map over the incoming tuple + // Example: `{ tup -> [ tup[0], [input: tup[1].output] ] + tup.drop(2) }` + map: null, + + // Apply a map over the ID element of a tuple (i.e. the first element) + // Example: `{ id -> id + "_foo" }` + mapId: null, + + // Apply a map over the data element of a tuple (i.e. the second element) + // Example: `{ data -> [ input: data.output ] }` + mapData: null, + + // Apply a map over the passthrough elements of a tuple (i.e. the tuple excl. the first two elements) + // Example: `{ pt -> pt.drop(1) }` + mapPassthrough: null, + + // Filter the channel + // Example: `{ tup -> tup[0] == "foo" }` + filter: null, + + // Choose whether or not to run the component on the tuple if the condition is true. + // Otherwise, the tuple will be passed through. + // Example: `{ tup -> tup[0] != "skip_this" }` + runIf: null, + + // Rename keys in the data field of the tuple (i.e. the second element) + // Will likely be deprecated in favour of `fromState`. + // Example: `[ "new_key": "old_key" ]` + renameKeys: null, + + // Fetch data from the state and pass it to the module without altering the current state. + // + // `fromState` should be `null`, `List[String]`, `Map[String, String]` or a function. + // + // - If it is `null`, the state will be passed to the module as is. + // - If it is a `List[String]`, the data will be the values of the state at the given keys. + // - If it is a `Map[String, String]`, the data will be the values of the state at the given keys, with the keys renamed according to the map. + // - If it is a function, the tuple (`[id, state]`) in the channel will be passed to the function, and the result will be used as the data. + // + // Example: `{ id, state -> [input: state.fastq_file] }` + // Default: `null` + fromState: null, + + // Determine how the state should be updated after the module has been run. + // + // `toState` should be `null`, `List[String]`, `Map[String, String]` or a function. + // + // - If it is `null`, the state will be replaced with the output of the module. + // - If it is a `List[String]`, the state will be updated with the values of the data at the given keys. + // - If it is a `Map[String, String]`, the state will be updated with the values of the data at the given keys, with the keys renamed according to the map. + // - If it is a function, a tuple (`[id, output, state]`) will be passed to the function, and the result will be used as the new state. + // + // Example: `{ id, output, state -> state + [counts: state.output] }` + // Default: `{ id, output, state -> output }` + toState: null, + + // Whether or not to print debug messages + // Default: `false` + debug: false +] + +// initialise default workflow +meta["workflow"] = workflowFactory([key: meta.config.name], meta.defaults, meta) + +// add workflow to environment +nextflow.script.ScriptMeta.current().addDefinition(meta.workflow) + +// anonymous workflow for running this module as a standalone +workflow { + // add id argument if it's not already in the config + // TODO: deep copy + def newConfig = deepClone(meta.config) + def newParams = deepClone(params) + + def argsContainsId = newConfig.allArguments.any{it.plainName == "id"} + if (!argsContainsId) { + def idArg = [ + 'name': '--id', + 'required': false, + 'type': 'string', + 'description': 'A unique id for every entry.', + 'multiple': false + ] + newConfig.arguments.add(0, idArg) + newConfig = processConfig(newConfig) + } + if (!newParams.containsKey("id")) { + newParams.id = "run" + } + + helpMessage(newConfig) + + channelFromParams(newParams, newConfig) + // make sure id is not in the state if id is not in the args + | map {id, state -> + if (!argsContainsId) { + [id, state.findAll{k, v -> k != "id"}] + } else { + [id, state] + } + } + | meta.workflow.run( + auto: [ publish: "state" ] + ) +} + +// END COMPONENT-SPECIFIC CODE diff --git a/target/nextflow/bedtools/bedtools_merge/nextflow.config b/target/nextflow/bedtools/bedtools_merge/nextflow.config new file mode 100644 index 00000000..517a6ef3 --- /dev/null +++ b/target/nextflow/bedtools/bedtools_merge/nextflow.config @@ -0,0 +1,126 @@ +manifest { + name = 'bedtools/bedtools_merge' + mainScript = 'main.nf' + nextflowVersion = '!>=20.12.1-edge' + version = 'main' + description = 'Merges overlapping BED/GFF/VCF entries into a single interval.\n' + author = 'Theodoro Gasperin Terra Camargo' +} + +process.container = 'nextflow/bash:latest' + +// detect tempdir +tempDir = java.nio.file.Paths.get( + System.getenv('NXF_TEMP') ?: + System.getenv('VIASH_TEMP') ?: + System.getenv('TEMPDIR') ?: + System.getenv('TMPDIR') ?: + '/tmp' +).toAbsolutePath() + +profiles { + no_publish { + process { + withName: '.*' { + publishDir = [ + enabled: false + ] + } + } + } + mount_temp { + docker.temp = tempDir + podman.temp = tempDir + charliecloud.temp = tempDir + } + docker { + docker.enabled = true + // docker.userEmulation = true + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + singularity { + singularity.enabled = true + singularity.autoMounts = true + docker.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + podman { + podman.enabled = true + docker.enabled = false + singularity.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + shifter { + shifter.enabled = true + docker.enabled = false + singularity.enabled = false + podman.enabled = false + charliecloud.enabled = false + } + charliecloud { + charliecloud.enabled = true + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + } +} + +process{ + withLabel: mem1gb { memory = 1000000000.B } + withLabel: mem2gb { memory = 2000000000.B } + withLabel: mem5gb { memory = 5000000000.B } + withLabel: mem10gb { memory = 10000000000.B } + withLabel: mem20gb { memory = 20000000000.B } + withLabel: mem50gb { memory = 50000000000.B } + withLabel: mem100gb { memory = 100000000000.B } + withLabel: mem200gb { memory = 200000000000.B } + withLabel: mem500gb { memory = 500000000000.B } + withLabel: mem1tb { memory = 1000000000000.B } + withLabel: mem2tb { memory = 2000000000000.B } + withLabel: mem5tb { memory = 5000000000000.B } + withLabel: mem10tb { memory = 10000000000000.B } + withLabel: mem20tb { memory = 20000000000000.B } + withLabel: mem50tb { memory = 50000000000000.B } + withLabel: mem100tb { memory = 100000000000000.B } + withLabel: mem200tb { memory = 200000000000000.B } + withLabel: mem500tb { memory = 500000000000000.B } + withLabel: mem1gib { memory = 1073741824.B } + withLabel: mem2gib { memory = 2147483648.B } + withLabel: mem4gib { memory = 4294967296.B } + withLabel: mem8gib { memory = 8589934592.B } + withLabel: mem16gib { memory = 17179869184.B } + withLabel: mem32gib { memory = 34359738368.B } + withLabel: mem64gib { memory = 68719476736.B } + withLabel: mem128gib { memory = 137438953472.B } + withLabel: mem256gib { memory = 274877906944.B } + withLabel: mem512gib { memory = 549755813888.B } + withLabel: mem1tib { memory = 1099511627776.B } + withLabel: mem2tib { memory = 2199023255552.B } + withLabel: mem4tib { memory = 4398046511104.B } + withLabel: mem8tib { memory = 8796093022208.B } + withLabel: mem16tib { memory = 17592186044416.B } + withLabel: mem32tib { memory = 35184372088832.B } + withLabel: mem64tib { memory = 70368744177664.B } + withLabel: mem128tib { memory = 140737488355328.B } + withLabel: mem256tib { memory = 281474976710656.B } + withLabel: mem512tib { memory = 562949953421312.B } + withLabel: cpu1 { cpus = 1 } + withLabel: cpu2 { cpus = 2 } + withLabel: cpu5 { cpus = 5 } + withLabel: cpu10 { cpus = 10 } + withLabel: cpu20 { cpus = 20 } + withLabel: cpu50 { cpus = 50 } + withLabel: cpu100 { cpus = 100 } + withLabel: cpu200 { cpus = 200 } + withLabel: cpu500 { cpus = 500 } + withLabel: cpu1000 { cpus = 1000 } +} + + diff --git a/target/nextflow/bedtools/bedtools_merge/nextflow_schema.json b/target/nextflow/bedtools/bedtools_merge/nextflow_schema.json new file mode 100644 index 00000000..be7b1f37 --- /dev/null +++ b/target/nextflow/bedtools/bedtools_merge/nextflow_schema.json @@ -0,0 +1,216 @@ +{ +"$schema": "http://json-schema.org/draft-07/schema", +"title": "bedtools_merge", +"description": "Merges overlapping BED/GFF/VCF entries into a single interval.\n", +"type": "object", +"definitions": { + + + + "inputs" : { + "title": "Inputs", + "type": "object", + "description": "No description", + "properties": { + + + "input": { + "type": + "string", + "description": "Type: `file`, required. Input file (BED/GFF/VCF) to be merged", + "help_text": "Type: `file`, required. Input file (BED/GFF/VCF) to be merged." + + } + + +} +}, + + + "outputs" : { + "title": "Outputs", + "type": "object", + "description": "No description", + "properties": { + + + "output": { + "type": + "string", + "description": "Type: `file`, required, default: `$id.$key.output.output`. Output merged file BED to be written", + "help_text": "Type: `file`, required, default: `$id.$key.output.output`. Output merged file BED to be written." + , + "default": "$id.$key.output.output" + } + + +} +}, + + + "options" : { + "title": "Options", + "type": "object", + "description": "No description", + "properties": { + + + "strand": { + "type": + "boolean", + "description": "Type: `boolean_true`, default: `false`. Force strandedness", + "help_text": "Type: `boolean_true`, default: `false`. Force strandedness. That is, only merge features\nthat are on the same strand.\n- By default, merging is done without respect to strand.\n" + , + "default": "False" + } + + + , + "specific_strand": { + "type": + "string", + "description": "Type: `string`, choices: ``+`, `-``. Force merge for one specific strand only", + "help_text": "Type: `string`, choices: ``+`, `-``. Force merge for one specific strand only.\nFollow with + or - to force merge from only\nthe forward or reverse strand, respectively.\n- By default, merging is done without respect to strand.\n", + "enum": ["+", "-"] + + + } + + + , + "distance": { + "type": + "integer", + "description": "Type: `integer`. Maximum distance between features allowed for features\nto be merged", + "help_text": "Type: `integer`. Maximum distance between features allowed for features\nto be merged.\n- Def. 0. That is, overlapping \u0026 book-ended features are merged.\n- (INTEGER)\n- Note: negative values enforce the number of b.p. required for overlap.\n" + + } + + + , + "columns": { + "type": + "integer", + "description": "Type: `integer`. Specify columns from the B file to map onto intervals in A", + "help_text": "Type: `integer`. Specify columns from the B file to map onto intervals in A.\nDefault: 5.\nMultiple columns can be specified in a comma-delimited list.\n" + + } + + + , + "operation": { + "type": + "string", + "description": "Type: `string`. Specify the operation that should be applied to -c", + "help_text": "Type: `string`. Specify the operation that should be applied to -c.\nValid operations:\n sum, min, max, absmin, absmax,\n mean, median, mode, antimode\n stdev, sstdev\n collapse (i.e., print a delimited list (duplicates allowed)), \n distinct (i.e., print a delimited list (NO duplicates allowed)), \n distinct_sort_num (as distinct, sorted numerically, ascending),\n distinct_sort_num_desc (as distinct, sorted numerically, desscending),\n distinct_only (delimited list of only unique values),\n count\n count_distinct (i.e., a count of the unique values in the column), \n first (i.e., just the first value in the column), \n last (i.e., just the last value in the column), \nDefault: sum\nMultiple operations can be specified in a comma-delimited list.\n\nIf there is only column, but multiple operations, all operations will be\napplied on that column. Likewise, if there is only one operation, but\nmultiple columns, that operation will be applied to all columns.\nOtherwise, the number of columns must match the the number of operations,\nand will be applied in respective order.\nE.g., \"-c 5,4,6 -o sum,mean,count\" will give the sum of column 5,\nthe mean of column 4, and the count of column 6.\nThe order of output columns will match the ordering given in the command.\n" + + } + + + , + "delimiter": { + "type": + "string", + "description": "Type: `string`, default: `,`, example: `|`. Specify a custom delimiter for the collapse operations", + "help_text": "Type: `string`, default: `,`, example: `|`. Specify a custom delimiter for the collapse operations.\n" + , + "default": "," + } + + + , + "precision": { + "type": + "integer", + "description": "Type: `integer`. Sets the decimal precision for output (Default: 5)", + "help_text": "Type: `integer`. Sets the decimal precision for output (Default: 5).\n" + + } + + + , + "bed": { + "type": + "boolean", + "description": "Type: `boolean_true`, default: `false`. If using BAM input, write output as BED", + "help_text": "Type: `boolean_true`, default: `false`. If using BAM input, write output as BED.\n" + , + "default": "False" + } + + + , + "header": { + "type": + "boolean", + "description": "Type: `boolean_true`, default: `false`. Print the header from the A file prior to results", + "help_text": "Type: `boolean_true`, default: `false`. Print the header from the A file prior to results.\n" + , + "default": "False" + } + + + , + "no_buffer": { + "type": + "boolean", + "description": "Type: `boolean_true`, default: `false`. Disable buffered output", + "help_text": "Type: `boolean_true`, default: `false`. Disable buffered output. Using this option will cause each line\nof output to be printed as it is generated, rather than saved\nin a buffer. This will make printing large output files \nnoticeably slower, but can be useful in conjunction with\nother software tools and scripts that need to process one\nline of bedtools output at a time.\n" + , + "default": "False" + } + + +} +}, + + + "nextflow input-output arguments" : { + "title": "Nextflow input-output arguments", + "type": "object", + "description": "Input/output parameters for Nextflow itself. Please note that both publishDir and publish_dir are supported but at least one has to be configured.", + "properties": { + + + "publish_dir": { + "type": + "string", + "description": "Type: `string`, required, example: `output/`. Path to an output directory", + "help_text": "Type: `string`, required, example: `output/`. Path to an output directory." + + } + + + , + "param_list": { + "type": + "string", + "description": "Type: `string`, example: `my_params.yaml`. Allows inputting multiple parameter sets to initialise a Nextflow channel", + "help_text": "Type: `string`, example: `my_params.yaml`. Allows inputting multiple parameter sets to initialise a Nextflow channel. A `param_list` can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob.\n\n* A list of maps (as-is) where the keys of each map corresponds to the arguments of the pipeline. Example: in a `nextflow.config` file: `param_list: [ [\u0027id\u0027: \u0027foo\u0027, \u0027input\u0027: \u0027foo.txt\u0027], [\u0027id\u0027: \u0027bar\u0027, \u0027input\u0027: \u0027bar.txt\u0027] ]`.\n* A csv file should have column names which correspond to the different arguments of this pipeline. Example: `--param_list data.csv` with columns `id,input`.\n* A json or a yaml file should be a list of maps, each of which has keys corresponding to the arguments of the pipeline. Example: `--param_list data.json` with contents `[ {\u0027id\u0027: \u0027foo\u0027, \u0027input\u0027: \u0027foo.txt\u0027}, {\u0027id\u0027: \u0027bar\u0027, \u0027input\u0027: \u0027bar.txt\u0027} ]`.\n* A yaml blob can also be passed directly as a string. Example: `--param_list \"[ {\u0027id\u0027: \u0027foo\u0027, \u0027input\u0027: \u0027foo.txt\u0027}, {\u0027id\u0027: \u0027bar\u0027, \u0027input\u0027: \u0027bar.txt\u0027} ]\"`.\n\nWhen passing a csv, json or yaml file, relative path names are relativized to the location of the parameter file. No relativation is performed when `param_list` is a list of maps (as-is) or a yaml blob.", + "hidden": true + + } + + +} +} +}, +"allOf": [ + + { + "$ref": "#/definitions/inputs" + }, + + { + "$ref": "#/definitions/outputs" + }, + + { + "$ref": "#/definitions/options" + }, + + { + "$ref": "#/definitions/nextflow input-output arguments" + } +] +} diff --git a/target/nextflow/bedtools/bedtools_sort/.config.vsh.yaml b/target/nextflow/bedtools/bedtools_sort/.config.vsh.yaml index b7b639a6..1975f327 100644 --- a/target/nextflow/bedtools/bedtools_sort/.config.vsh.yaml +++ b/target/nextflow/bedtools/bedtools_sort/.config.vsh.yaml @@ -221,15 +221,15 @@ build_info: engine: "docker|native" output: "target/nextflow/bedtools/bedtools_sort" executable: "target/nextflow/bedtools/bedtools_sort/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/bedtools/bedtools_sort/main.nf b/target/nextflow/bedtools/bedtools_sort/main.nf index ef6322d6..1a28ff8b 100644 --- a/target/nextflow/bedtools/bedtools_sort/main.nf +++ b/target/nextflow/bedtools/bedtools_sort/main.nf @@ -1,6 +1,6 @@ // bedtools_sort main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -763,8 +763,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -786,7 +789,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -824,8 +840,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1601,8 +1620,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2629,30 +2648,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2723,7 +2743,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -3065,15 +3085,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/bedtools/bedtools_sort", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/busco/busco_download_datasets/.config.vsh.yaml b/target/nextflow/busco/busco_download_datasets/.config.vsh.yaml index 0bf38cc9..77ddab90 100644 --- a/target/nextflow/busco/busco_download_datasets/.config.vsh.yaml +++ b/target/nextflow/busco/busco_download_datasets/.config.vsh.yaml @@ -157,15 +157,15 @@ build_info: engine: "docker|native" output: "target/nextflow/busco/busco_download_datasets" executable: "target/nextflow/busco/busco_download_datasets/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/busco/busco_download_datasets/main.nf b/target/nextflow/busco/busco_download_datasets/main.nf index 02a60944..877a758f 100644 --- a/target/nextflow/busco/busco_download_datasets/main.nf +++ b/target/nextflow/busco/busco_download_datasets/main.nf @@ -1,6 +1,6 @@ // busco_download_datasets main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -763,8 +763,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -786,7 +789,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -824,8 +840,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1601,8 +1620,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2629,30 +2648,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2723,7 +2743,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -2984,15 +3004,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/busco/busco_download_datasets", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/busco/busco_list_datasets/.config.vsh.yaml b/target/nextflow/busco/busco_list_datasets/.config.vsh.yaml index 37b6716b..43c78aaa 100644 --- a/target/nextflow/busco/busco_list_datasets/.config.vsh.yaml +++ b/target/nextflow/busco/busco_list_datasets/.config.vsh.yaml @@ -144,15 +144,15 @@ build_info: engine: "docker|native" output: "target/nextflow/busco/busco_list_datasets" executable: "target/nextflow/busco/busco_list_datasets/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/busco/busco_list_datasets/main.nf b/target/nextflow/busco/busco_list_datasets/main.nf index fc971cb4..3eb3612c 100644 --- a/target/nextflow/busco/busco_list_datasets/main.nf +++ b/target/nextflow/busco/busco_list_datasets/main.nf @@ -1,6 +1,6 @@ // busco_list_datasets main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -763,8 +763,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -786,7 +789,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -824,8 +840,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1601,8 +1620,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2629,30 +2648,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2723,7 +2743,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -2970,15 +2990,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/busco/busco_list_datasets", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/busco/busco_run/.config.vsh.yaml b/target/nextflow/busco/busco_run/.config.vsh.yaml index a8fadb6b..43af0b95 100644 --- a/target/nextflow/busco/busco_run/.config.vsh.yaml +++ b/target/nextflow/busco/busco_run/.config.vsh.yaml @@ -422,15 +422,15 @@ build_info: engine: "docker|native" output: "target/nextflow/busco/busco_run" executable: "target/nextflow/busco/busco_run/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/busco/busco_run/main.nf b/target/nextflow/busco/busco_run/main.nf index 878dc89d..2d11dab6 100644 --- a/target/nextflow/busco/busco_run/main.nf +++ b/target/nextflow/busco/busco_run/main.nf @@ -1,6 +1,6 @@ // busco_run main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -763,8 +763,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -786,7 +789,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -824,8 +840,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1601,8 +1620,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2629,30 +2648,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2723,7 +2743,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -3282,15 +3302,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/busco/busco_run", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/busco/busco_run/nextflow_schema.json b/target/nextflow/busco/busco_run/nextflow_schema.json index 64390d89..eabf60cb 100644 --- a/target/nextflow/busco/busco_run/nextflow_schema.json +++ b/target/nextflow/busco/busco_run/nextflow_schema.json @@ -103,8 +103,8 @@ "output_dir": { "type": "string", - "description": "Type: `file`, default: `$id.$key.output_dir.output_dir`, example: `output_dir/`. The full output directory, if so desired", - "help_text": "Type: `file`, default: `$id.$key.output_dir.output_dir`, example: `output_dir/`. The full output directory, if so desired.\n" + "description": "Type: `file`, default: `$id.$key.output_dir.output_dir`, example: `output_dir`. The full output directory, if so desired", + "help_text": "Type: `file`, default: `$id.$key.output_dir.output_dir`, example: `output_dir`. The full output directory, if so desired.\n" , "default": "$id.$key.output_dir.output_dir" } diff --git a/target/nextflow/cutadapt/.config.vsh.yaml b/target/nextflow/cutadapt/.config.vsh.yaml index 488c11c7..22685fc1 100644 --- a/target/nextflow/cutadapt/.config.vsh.yaml +++ b/target/nextflow/cutadapt/.config.vsh.yaml @@ -739,15 +739,15 @@ build_info: engine: "docker|native" output: "target/nextflow/cutadapt" executable: "target/nextflow/cutadapt/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/cutadapt/main.nf b/target/nextflow/cutadapt/main.nf index a9a17e2e..0c3b21db 100644 --- a/target/nextflow/cutadapt/main.nf +++ b/target/nextflow/cutadapt/main.nf @@ -1,6 +1,6 @@ // cutadapt main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -763,8 +763,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -786,7 +789,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -824,8 +840,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1601,8 +1620,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2629,30 +2648,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2723,7 +2743,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -3592,15 +3612,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/cutadapt", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/cutadapt/nextflow_schema.json b/target/nextflow/cutadapt/nextflow_schema.json index d5f821f0..20301884 100644 --- a/target/nextflow/cutadapt/nextflow_schema.json +++ b/target/nextflow/cutadapt/nextflow_schema.json @@ -17,8 +17,8 @@ "adapter": { "type": "string", - "description": "Type: List of `string`, multiple_sep: `\":\"`. Sequence of an adapter ligated to the 3\u0027 end (paired data:\nof the first read)", - "help_text": "Type: List of `string`, multiple_sep: `\":\"`. Sequence of an adapter ligated to the 3\u0027 end (paired data:\nof the first read). The adapter and subsequent bases are\ntrimmed. If a \u0027$\u0027 character is appended (\u0027anchoring\u0027), the\nadapter is only found if it is a suffix of the read.\n" + "description": "Type: List of `string`, multiple_sep: `\";\"`. Sequence of an adapter ligated to the 3\u0027 end (paired data:\nof the first read)", + "help_text": "Type: List of `string`, multiple_sep: `\";\"`. Sequence of an adapter ligated to the 3\u0027 end (paired data:\nof the first read). The adapter and subsequent bases are\ntrimmed. If a \u0027$\u0027 character is appended (\u0027anchoring\u0027), the\nadapter is only found if it is a suffix of the read.\n" } @@ -27,8 +27,8 @@ "front": { "type": "string", - "description": "Type: List of `string`, multiple_sep: `\":\"`. Sequence of an adapter ligated to the 5\u0027 end (paired data:\nof the first read)", - "help_text": "Type: List of `string`, multiple_sep: `\":\"`. Sequence of an adapter ligated to the 5\u0027 end (paired data:\nof the first read). The adapter and any preceding bases\nare trimmed. Partial matches at the 5\u0027 end are allowed. If\na \u0027^\u0027 character is prepended (\u0027anchoring\u0027), the adapter is\nonly found if it is a prefix of the read.\n" + "description": "Type: List of `string`, multiple_sep: `\";\"`. Sequence of an adapter ligated to the 5\u0027 end (paired data:\nof the first read)", + "help_text": "Type: List of `string`, multiple_sep: `\";\"`. Sequence of an adapter ligated to the 5\u0027 end (paired data:\nof the first read). The adapter and any preceding bases\nare trimmed. Partial matches at the 5\u0027 end are allowed. If\na \u0027^\u0027 character is prepended (\u0027anchoring\u0027), the adapter is\nonly found if it is a prefix of the read.\n" } @@ -37,8 +37,8 @@ "anywhere": { "type": "string", - "description": "Type: List of `string`, multiple_sep: `\":\"`. Sequence of an adapter that may be ligated to the 5\u0027 or 3\u0027\nend (paired data: of the first read)", - "help_text": "Type: List of `string`, multiple_sep: `\":\"`. Sequence of an adapter that may be ligated to the 5\u0027 or 3\u0027\nend (paired data: of the first read). Both types of\nmatches as described under -a and -g are allowed. If the\nfirst base of the read is part of the match, the behavior\nis as with -g, otherwise as with -a. This option is mostly\nfor rescuing failed library preparations - do not use if\nyou know which end your adapter was ligated to!\n" + "description": "Type: List of `string`, multiple_sep: `\";\"`. Sequence of an adapter that may be ligated to the 5\u0027 or 3\u0027\nend (paired data: of the first read)", + "help_text": "Type: List of `string`, multiple_sep: `\";\"`. Sequence of an adapter that may be ligated to the 5\u0027 or 3\u0027\nend (paired data: of the first read). Both types of\nmatches as described under -a and -g are allowed. If the\nfirst base of the read is part of the match, the behavior\nis as with -g, otherwise as with -a. This option is mostly\nfor rescuing failed library preparations - do not use if\nyou know which end your adapter was ligated to!\n" } @@ -57,8 +57,8 @@ "adapter_fasta": { "type": "string", - "description": "Type: List of `file`, multiple_sep: `\":\"`. Fasta file containing sequences of an adapter ligated to the 3\u0027 end (paired data:\nof the first read)", - "help_text": "Type: List of `file`, multiple_sep: `\":\"`. Fasta file containing sequences of an adapter ligated to the 3\u0027 end (paired data:\nof the first read). The adapter and subsequent bases are\ntrimmed. If a \u0027$\u0027 character is appended (\u0027anchoring\u0027), the\nadapter is only found if it is a suffix of the read.\n" + "description": "Type: List of `file`, multiple_sep: `\";\"`. Fasta file containing sequences of an adapter ligated to the 3\u0027 end (paired data:\nof the first read)", + "help_text": "Type: List of `file`, multiple_sep: `\";\"`. Fasta file containing sequences of an adapter ligated to the 3\u0027 end (paired data:\nof the first read). The adapter and subsequent bases are\ntrimmed. If a \u0027$\u0027 character is appended (\u0027anchoring\u0027), the\nadapter is only found if it is a suffix of the read.\n" } @@ -97,8 +97,8 @@ "adapter_r2": { "type": "string", - "description": "Type: List of `string`, multiple_sep: `\":\"`. Sequence of an adapter ligated to the 3\u0027 end (paired data:\nof the first read)", - "help_text": "Type: List of `string`, multiple_sep: `\":\"`. Sequence of an adapter ligated to the 3\u0027 end (paired data:\nof the first read). The adapter and subsequent bases are\ntrimmed. If a \u0027$\u0027 character is appended (\u0027anchoring\u0027), the\nadapter is only found if it is a suffix of the read.\n" + "description": "Type: List of `string`, multiple_sep: `\";\"`. Sequence of an adapter ligated to the 3\u0027 end (paired data:\nof the first read)", + "help_text": "Type: List of `string`, multiple_sep: `\";\"`. Sequence of an adapter ligated to the 3\u0027 end (paired data:\nof the first read). The adapter and subsequent bases are\ntrimmed. If a \u0027$\u0027 character is appended (\u0027anchoring\u0027), the\nadapter is only found if it is a suffix of the read.\n" } @@ -107,8 +107,8 @@ "front_r2": { "type": "string", - "description": "Type: List of `string`, multiple_sep: `\":\"`. Sequence of an adapter ligated to the 5\u0027 end (paired data:\nof the first read)", - "help_text": "Type: List of `string`, multiple_sep: `\":\"`. Sequence of an adapter ligated to the 5\u0027 end (paired data:\nof the first read). The adapter and any preceding bases\nare trimmed. Partial matches at the 5\u0027 end are allowed. If\na \u0027^\u0027 character is prepended (\u0027anchoring\u0027), the adapter is\nonly found if it is a prefix of the read.\n" + "description": "Type: List of `string`, multiple_sep: `\";\"`. Sequence of an adapter ligated to the 5\u0027 end (paired data:\nof the first read)", + "help_text": "Type: List of `string`, multiple_sep: `\";\"`. Sequence of an adapter ligated to the 5\u0027 end (paired data:\nof the first read). The adapter and any preceding bases\nare trimmed. Partial matches at the 5\u0027 end are allowed. If\na \u0027^\u0027 character is prepended (\u0027anchoring\u0027), the adapter is\nonly found if it is a prefix of the read.\n" } @@ -117,8 +117,8 @@ "anywhere_r2": { "type": "string", - "description": "Type: List of `string`, multiple_sep: `\":\"`. Sequence of an adapter that may be ligated to the 5\u0027 or 3\u0027\nend (paired data: of the first read)", - "help_text": "Type: List of `string`, multiple_sep: `\":\"`. Sequence of an adapter that may be ligated to the 5\u0027 or 3\u0027\nend (paired data: of the first read). Both types of\nmatches as described under -a and -g are allowed. If the\nfirst base of the read is part of the match, the behavior\nis as with -g, otherwise as with -a. This option is mostly\nfor rescuing failed library preparations - do not use if\nyou know which end your adapter was ligated to!\n" + "description": "Type: List of `string`, multiple_sep: `\";\"`. Sequence of an adapter that may be ligated to the 5\u0027 or 3\u0027\nend (paired data: of the first read)", + "help_text": "Type: List of `string`, multiple_sep: `\";\"`. Sequence of an adapter that may be ligated to the 5\u0027 or 3\u0027\nend (paired data: of the first read). Both types of\nmatches as described under -a and -g are allowed. If the\nfirst base of the read is part of the match, the behavior\nis as with -g, otherwise as with -a. This option is mostly\nfor rescuing failed library preparations - do not use if\nyou know which end your adapter was ligated to!\n" } @@ -359,8 +359,8 @@ "cut": { "type": "string", - "description": "Type: List of `integer`, multiple_sep: `\":\"`. Remove LEN bases from each read (or R1 if paired; use --cut_r2\noption for R2)", - "help_text": "Type: List of `integer`, multiple_sep: `\":\"`. Remove LEN bases from each read (or R1 if paired; use --cut_r2\noption for R2). If LEN is positive, remove bases from the\nbeginning. If LEN is negative, remove bases from the end.\nCan be used twice if LENs have different signs. Applied\n*before* adapter trimming.\n" + "description": "Type: List of `integer`, multiple_sep: `\";\"`. Remove LEN bases from each read (or R1 if paired; use --cut_r2\noption for R2)", + "help_text": "Type: List of `integer`, multiple_sep: `\";\"`. Remove LEN bases from each read (or R1 if paired; use --cut_r2\noption for R2). If LEN is positive, remove bases from the\nbeginning. If LEN is negative, remove bases from the end.\nCan be used twice if LENs have different signs. Applied\n*before* adapter trimming.\n" } @@ -369,8 +369,8 @@ "cut_r2": { "type": "string", - "description": "Type: List of `integer`, multiple_sep: `\":\"`. Remove LEN bases from each read (for R2)", - "help_text": "Type: List of `integer`, multiple_sep: `\":\"`. Remove LEN bases from each read (for R2). If LEN is positive, remove bases from the\nbeginning. If LEN is negative, remove bases from the end.\nCan be used twice if LENs have different signs. Applied\n*before* adapter trimming.\n" + "description": "Type: List of `integer`, multiple_sep: `\";\"`. Remove LEN bases from each read (for R2)", + "help_text": "Type: List of `integer`, multiple_sep: `\";\"`. Remove LEN bases from each read (for R2). If LEN is positive, remove bases from the\nbeginning. If LEN is negative, remove bases from the end.\nCan be used twice if LENs have different signs. Applied\n*before* adapter trimming.\n" } @@ -638,8 +638,8 @@ "output": { "type": "string", - "description": "Type: List of `file`, required, default: `$id.$key.output_*.fast[a,q]`, example: `fastq/*_001.fast[a,q]`, multiple_sep: `\":\"`. Glob pattern for matching the expected output files", - "help_text": "Type: List of `file`, required, default: `$id.$key.output_*.fast[a,q]`, example: `fastq/*_001.fast[a,q]`, multiple_sep: `\":\"`. Glob pattern for matching the expected output files.\nShould include `$output_dir`.\n" + "description": "Type: List of `file`, required, default: `$id.$key.output_*.fast[a,q]`, example: `fastq/*_001.fast[a,q]`, multiple_sep: `\";\"`. Glob pattern for matching the expected output files", + "help_text": "Type: List of `file`, required, default: `$id.$key.output_*.fast[a,q]`, example: `fastq/*_001.fast[a,q]`, multiple_sep: `\";\"`. Glob pattern for matching the expected output files.\nShould include `$output_dir`.\n" , "default": "$id.$key.output_*.fast[a,q]" } diff --git a/target/nextflow/falco/.config.vsh.yaml b/target/nextflow/falco/.config.vsh.yaml index ec4bbf23..87e8ab21 100644 --- a/target/nextflow/falco/.config.vsh.yaml +++ b/target/nextflow/falco/.config.vsh.yaml @@ -316,15 +316,15 @@ build_info: engine: "docker|native" output: "target/nextflow/falco" executable: "target/nextflow/falco/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/falco/main.nf b/target/nextflow/falco/main.nf index 6acc35c7..b880f7a8 100644 --- a/target/nextflow/falco/main.nf +++ b/target/nextflow/falco/main.nf @@ -1,6 +1,6 @@ // falco main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -763,8 +763,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -786,7 +789,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -824,8 +840,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1601,8 +1620,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2629,30 +2648,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2723,7 +2743,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -3143,15 +3163,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/falco", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/falco/nextflow_schema.json b/target/nextflow/falco/nextflow_schema.json index 71b5872f..22f431e3 100644 --- a/target/nextflow/falco/nextflow_schema.json +++ b/target/nextflow/falco/nextflow_schema.json @@ -17,8 +17,8 @@ "input": { "type": "string", - "description": "Type: List of `file`, required, example: `input1.fastq;input2.fastq`, multiple_sep: `\":\"`. input fastq files", - "help_text": "Type: List of `file`, required, example: `input1.fastq;input2.fastq`, multiple_sep: `\":\"`. input fastq files" + "description": "Type: List of `file`, required, example: `input1.fastq;input2.fastq`, multiple_sep: `\";\"`. input fastq files", + "help_text": "Type: List of `file`, required, example: `input1.fastq;input2.fastq`, multiple_sep: `\";\"`. input fastq files" } diff --git a/target/nextflow/fastp/.config.vsh.yaml b/target/nextflow/fastp/.config.vsh.yaml index 78fd615e..9ed77890 100644 --- a/target/nextflow/fastp/.config.vsh.yaml +++ b/target/nextflow/fastp/.config.vsh.yaml @@ -1082,15 +1082,15 @@ build_info: engine: "docker|native" output: "target/nextflow/fastp" executable: "target/nextflow/fastp/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/fastp/main.nf b/target/nextflow/fastp/main.nf index b5493232..fe9be6f6 100644 --- a/target/nextflow/fastp/main.nf +++ b/target/nextflow/fastp/main.nf @@ -1,6 +1,6 @@ // fastp main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -763,8 +763,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -786,7 +789,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -824,8 +840,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1601,8 +1620,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2629,30 +2648,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2723,7 +2743,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -3996,15 +4016,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/fastp", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/featurecounts/.config.vsh.yaml b/target/nextflow/featurecounts/.config.vsh.yaml index 0a48a279..259d682d 100644 --- a/target/nextflow/featurecounts/.config.vsh.yaml +++ b/target/nextflow/featurecounts/.config.vsh.yaml @@ -644,15 +644,15 @@ build_info: engine: "docker|native" output: "target/nextflow/featurecounts" executable: "target/nextflow/featurecounts/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/featurecounts/main.nf b/target/nextflow/featurecounts/main.nf index f140dca6..9cd41acf 100644 --- a/target/nextflow/featurecounts/main.nf +++ b/target/nextflow/featurecounts/main.nf @@ -1,6 +1,6 @@ // featurecounts main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -763,8 +763,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -786,7 +789,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -824,8 +840,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1601,8 +1620,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2629,30 +2648,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2723,7 +2743,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -3522,15 +3542,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/featurecounts", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/featurecounts/nextflow_schema.json b/target/nextflow/featurecounts/nextflow_schema.json index 564393b1..d55c22dd 100644 --- a/target/nextflow/featurecounts/nextflow_schema.json +++ b/target/nextflow/featurecounts/nextflow_schema.json @@ -27,8 +27,8 @@ "input": { "type": "string", - "description": "Type: List of `file`, required, example: `input_file1.bam`, multiple_sep: `\":\"`. A list of SAM or BAM format files separated by semi-colon (;)", - "help_text": "Type: List of `file`, required, example: `input_file1.bam`, multiple_sep: `\":\"`. A list of SAM or BAM format files separated by semi-colon (;). They can be either name or location sorted. Location-sorted paired-end reads are automatically sorted by read names.\n" + "description": "Type: List of `file`, required, example: `input_file1.bam`, multiple_sep: `\";\"`. A list of SAM or BAM format files separated by semi-colon (;)", + "help_text": "Type: List of `file`, required, example: `input_file1.bam`, multiple_sep: `\";\"`. A list of SAM or BAM format files separated by semi-colon (;). They can be either name or location sorted. Location-sorted paired-end reads are automatically sorted by read names.\n" } @@ -102,8 +102,8 @@ "feature_type": { "type": "string", - "description": "Type: List of `string`, example: `exon`, multiple_sep: `\":\"`. Specify feature type(s) in a GTF annotation", - "help_text": "Type: List of `string`, example: `exon`, multiple_sep: `\":\"`. Specify feature type(s) in a GTF annotation. If multiple types are provided, they should be separated by \u0027;\u0027 with no space in between. \u0027exon\u0027 by default. Rows in the annotation with a matched feature will be extracted and used for read mapping.\n" + "description": "Type: List of `string`, example: `exon`, multiple_sep: `\";\"`. Specify feature type(s) in a GTF annotation", + "help_text": "Type: List of `string`, example: `exon`, multiple_sep: `\";\"`. Specify feature type(s) in a GTF annotation. If multiple types are provided, they should be separated by \u0027;\u0027 with no space in between. \u0027exon\u0027 by default. Rows in the annotation with a matched feature will be extracted and used for read mapping.\n" } @@ -122,8 +122,8 @@ "extra_attributes": { "type": "string", - "description": "Type: List of `string`, multiple_sep: `\":\"`. Extract extra attribute types from the provided GTF annotation and include them in the counting output", - "help_text": "Type: List of `string`, multiple_sep: `\":\"`. Extract extra attribute types from the provided GTF annotation and include them in the counting output. These attribute types will not be used to group features. If more than one attribute type is provided they should be separated by semicolon (;).\n" + "description": "Type: List of `string`, multiple_sep: `\";\"`. Extract extra attribute types from the provided GTF annotation and include them in the counting output", + "help_text": "Type: List of `string`, multiple_sep: `\";\"`. Extract extra attribute types from the provided GTF annotation and include them in the counting output. These attribute types will not be used to group features. If more than one attribute type is provided they should be separated by semicolon (;).\n" } @@ -194,8 +194,8 @@ "frac_overlap": { "type": "number", - "description": "Type: `double`, example: `0`. Minimum fraction of overlapping bases in a read that is required for read assignment", - "help_text": "Type: `double`, example: `0`. Minimum fraction of overlapping bases in a read that is required for read assignment. Value should be within range [0,1]. 0 by default. Number of overlapping bases is counted from both reads if paired end. Both this option and \u0027--min_overlap\u0027 option need to be satisfied for read assignment.\n" + "description": "Type: `double`, example: `0.0`. Minimum fraction of overlapping bases in a read that is required for read assignment", + "help_text": "Type: `double`, example: `0.0`. Minimum fraction of overlapping bases in a read that is required for read assignment. Value should be within range [0,1]. 0 by default. Number of overlapping bases is counted from both reads if paired end. Both this option and \u0027--min_overlap\u0027 option need to be satisfied for read assignment.\n" } @@ -204,8 +204,8 @@ "frac_overlap_feature": { "type": "number", - "description": "Type: `double`, example: `0`. Minimum fraction of overlapping bases in a feature that is required for read assignment", - "help_text": "Type: `double`, example: `0`. Minimum fraction of overlapping bases in a feature that is required for read assignment. Value should be within range [0,1]. 0 by default.\n" + "description": "Type: `double`, example: `0.0`. Minimum fraction of overlapping bases in a feature that is required for read assignment", + "help_text": "Type: `double`, example: `0.0`. Minimum fraction of overlapping bases in a feature that is required for read assignment. Value should be within range [0,1]. 0 by default.\n" } @@ -573,8 +573,8 @@ "detailed_results": { "type": "string", - "description": "Type: `file`, default: `$id.$key.detailed_results.detailed_results`, example: `detailed_results/`. Directory to save the detailed assignment results", - "help_text": "Type: `file`, default: `$id.$key.detailed_results.detailed_results`, example: `detailed_results/`. Directory to save the detailed assignment results. Use `--detailed_results_format` to determine the format of the detailed results.\n" + "description": "Type: `file`, default: `$id.$key.detailed_results.detailed_results`, example: `detailed_results`. Directory to save the detailed assignment results", + "help_text": "Type: `file`, default: `$id.$key.detailed_results.detailed_results`, example: `detailed_results`. Directory to save the detailed assignment results. Use `--detailed_results_format` to determine the format of the detailed results.\n" , "default": "$id.$key.detailed_results.detailed_results" } diff --git a/target/nextflow/gffread/.config.vsh.yaml b/target/nextflow/gffread/.config.vsh.yaml index d7099025..2bd500b0 100644 --- a/target/nextflow/gffread/.config.vsh.yaml +++ b/target/nextflow/gffread/.config.vsh.yaml @@ -684,15 +684,15 @@ build_info: engine: "docker|native" output: "target/nextflow/gffread" executable: "target/nextflow/gffread/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/gffread/main.nf b/target/nextflow/gffread/main.nf index 1ef70a9e..e241014a 100644 --- a/target/nextflow/gffread/main.nf +++ b/target/nextflow/gffread/main.nf @@ -1,6 +1,6 @@ // gffread main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -763,8 +763,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -786,7 +789,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -824,8 +840,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1601,8 +1620,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2629,30 +2648,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2723,7 +2743,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -3579,15 +3599,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/gffread", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/gffread/nextflow_schema.json b/target/nextflow/gffread/nextflow_schema.json index 670b645b..5d494634 100644 --- a/target/nextflow/gffread/nextflow_schema.json +++ b/target/nextflow/gffread/nextflow_schema.json @@ -272,8 +272,8 @@ "table": { "type": "string", - "description": "Type: List of `string`, multiple_sep: `\":\"`. Output a simple tab delimited format instead of GFF, with columns having the values \nof GFF attributes given in \u003cattrlist\u003e; special pseudo-attributes (prefixed by @) are \nrecognized:\n @id, @geneid, @chr, @start, @end, @strand, @numexons, @exons, @cds, @covlen, @cdslen\nIf any of --spliced_exons/--tr_cds/--spliced_cds FASTA output files are enabled, the \nsame fields (excluding @id) are appended to the definition line of corresponding FASTA\nrecords", - "help_text": "Type: List of `string`, multiple_sep: `\":\"`. Output a simple tab delimited format instead of GFF, with columns having the values \nof GFF attributes given in \u003cattrlist\u003e; special pseudo-attributes (prefixed by @) are \nrecognized:\n @id, @geneid, @chr, @start, @end, @strand, @numexons, @exons, @cds, @covlen, @cdslen\nIf any of --spliced_exons/--tr_cds/--spliced_cds FASTA output files are enabled, the \nsame fields (excluding @id) are appended to the definition line of corresponding FASTA\nrecords.\n" + "description": "Type: List of `string`, multiple_sep: `\";\"`. Output a simple tab delimited format instead of GFF, with columns having the values \nof GFF attributes given in \u003cattrlist\u003e; special pseudo-attributes (prefixed by @) are \nrecognized:\n @id, @geneid, @chr, @start, @end, @strand, @numexons, @exons, @cds, @covlen, @cdslen\nIf any of --spliced_exons/--tr_cds/--spliced_cds FASTA output files are enabled, the \nsame fields (excluding @id) are appended to the definition line of corresponding FASTA\nrecords", + "help_text": "Type: List of `string`, multiple_sep: `\";\"`. Output a simple tab delimited format instead of GFF, with columns having the values \nof GFF attributes given in \u003cattrlist\u003e; special pseudo-attributes (prefixed by @) are \nrecognized:\n @id, @geneid, @chr, @start, @end, @strand, @numexons, @exons, @cds, @covlen, @cdslen\nIf any of --spliced_exons/--tr_cds/--spliced_cds FASTA output files are enabled, the \nsame fields (excluding @id) are appended to the definition line of corresponding FASTA\nrecords.\n" } diff --git a/target/nextflow/lofreq/lofreq_call/.config.vsh.yaml b/target/nextflow/lofreq/lofreq_call/.config.vsh.yaml index 4ccd35a5..ec74a86e 100644 --- a/target/nextflow/lofreq/lofreq_call/.config.vsh.yaml +++ b/target/nextflow/lofreq/lofreq_call/.config.vsh.yaml @@ -506,15 +506,15 @@ build_info: engine: "docker|native" output: "target/nextflow/lofreq/lofreq_call" executable: "target/nextflow/lofreq/lofreq_call/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/lofreq/lofreq_call/main.nf b/target/nextflow/lofreq/lofreq_call/main.nf index 48c0387f..ee569893 100644 --- a/target/nextflow/lofreq/lofreq_call/main.nf +++ b/target/nextflow/lofreq/lofreq_call/main.nf @@ -1,6 +1,6 @@ // lofreq_call main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -763,8 +763,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -786,7 +789,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -824,8 +840,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1601,8 +1620,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2629,30 +2648,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2723,7 +2743,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -3387,15 +3407,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/lofreq/lofreq_call", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/lofreq/lofreq_indelqual/.config.vsh.yaml b/target/nextflow/lofreq/lofreq_indelqual/.config.vsh.yaml index b3d80488..776d966d 100644 --- a/target/nextflow/lofreq/lofreq_indelqual/.config.vsh.yaml +++ b/target/nextflow/lofreq/lofreq_indelqual/.config.vsh.yaml @@ -214,15 +214,15 @@ build_info: engine: "docker|native" output: "target/nextflow/lofreq/lofreq_indelqual" executable: "target/nextflow/lofreq/lofreq_indelqual/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/lofreq/lofreq_indelqual/main.nf b/target/nextflow/lofreq/lofreq_indelqual/main.nf index 88188ad0..c2565562 100644 --- a/target/nextflow/lofreq/lofreq_indelqual/main.nf +++ b/target/nextflow/lofreq/lofreq_indelqual/main.nf @@ -1,6 +1,6 @@ // lofreq_indelqual main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -763,8 +763,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -786,7 +789,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -824,8 +840,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1601,8 +1620,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2629,30 +2648,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2723,7 +2743,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -3050,15 +3070,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/lofreq/lofreq_indelqual", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/multiqc/.config.vsh.yaml b/target/nextflow/multiqc/.config.vsh.yaml index 8bfe5fc0..6e2042bb 100644 --- a/target/nextflow/multiqc/.config.vsh.yaml +++ b/target/nextflow/multiqc/.config.vsh.yaml @@ -455,15 +455,15 @@ build_info: engine: "docker|native" output: "target/nextflow/multiqc" executable: "target/nextflow/multiqc/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/multiqc/main.nf b/target/nextflow/multiqc/main.nf index 010f178c..c1f9eb67 100644 --- a/target/nextflow/multiqc/main.nf +++ b/target/nextflow/multiqc/main.nf @@ -1,6 +1,6 @@ // multiqc main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -763,8 +763,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -786,7 +789,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -824,8 +840,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1601,8 +1620,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2629,30 +2648,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2723,7 +2743,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -3339,15 +3359,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/multiqc", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/multiqc/nextflow_schema.json b/target/nextflow/multiqc/nextflow_schema.json index bd30c250..5f3f25eb 100644 --- a/target/nextflow/multiqc/nextflow_schema.json +++ b/target/nextflow/multiqc/nextflow_schema.json @@ -17,8 +17,8 @@ "input": { "type": "string", - "description": "Type: List of `file`, required, example: `data/results/`, multiple_sep: `\":\"`. File paths to be searched for analysis results to be included in the report", - "help_text": "Type: List of `file`, required, example: `data/results/`, multiple_sep: `\":\"`. File paths to be searched for analysis results to be included in the report.\n" + "description": "Type: List of `file`, required, example: `data/results`, multiple_sep: `\";\"`. File paths to be searched for analysis results to be included in the report", + "help_text": "Type: List of `file`, required, example: `data/results`, multiple_sep: `\";\"`. File paths to be searched for analysis results to be included in the report.\n" } @@ -80,8 +80,8 @@ "include_modules": { "type": "string", - "description": "Type: List of `string`, example: `fastqc:cutadapt`, multiple_sep: `\":\"`. Use only these module", - "help_text": "Type: List of `string`, example: `fastqc:cutadapt`, multiple_sep: `\":\"`. Use only these module" + "description": "Type: List of `string`, example: `fastqc;cutadapt`, multiple_sep: `\";\"`. Use only these module", + "help_text": "Type: List of `string`, example: `fastqc;cutadapt`, multiple_sep: `\";\"`. Use only these module" } @@ -90,8 +90,8 @@ "exclude_modules": { "type": "string", - "description": "Type: List of `string`, example: `fastqc:cutadapt`, multiple_sep: `\":\"`. Do not use only these modules", - "help_text": "Type: List of `string`, example: `fastqc:cutadapt`, multiple_sep: `\":\"`. Do not use only these modules" + "description": "Type: List of `string`, example: `fastqc;cutadapt`, multiple_sep: `\";\"`. Do not use only these modules", + "help_text": "Type: List of `string`, example: `fastqc;cutadapt`, multiple_sep: `\";\"`. Do not use only these modules" } @@ -100,8 +100,8 @@ "ignore_analysis": { "type": "string", - "description": "Type: List of `string`, example: `run_one/*:run_two/*`, multiple_sep: `\":\"`. ", - "help_text": "Type: List of `string`, example: `run_one/*:run_two/*`, multiple_sep: `\":\"`. " + "description": "Type: List of `string`, example: `run_one/*;run_two/*`, multiple_sep: `\";\"`. ", + "help_text": "Type: List of `string`, example: `run_one/*;run_two/*`, multiple_sep: `\";\"`. " } @@ -110,8 +110,8 @@ "ignore_samples": { "type": "string", - "description": "Type: List of `string`, example: `sample_1*:sample_3*`, multiple_sep: `\":\"`. ", - "help_text": "Type: List of `string`, example: `sample_1*:sample_3*`, multiple_sep: `\":\"`. " + "description": "Type: List of `string`, example: `sample_1*;sample_3*`, multiple_sep: `\";\"`. ", + "help_text": "Type: List of `string`, example: `sample_1*;sample_3*`, multiple_sep: `\";\"`. " } diff --git a/target/nextflow/pear/.config.vsh.yaml b/target/nextflow/pear/.config.vsh.yaml index 9e76fcb4..5d03d84f 100644 --- a/target/nextflow/pear/.config.vsh.yaml +++ b/target/nextflow/pear/.config.vsh.yaml @@ -397,15 +397,15 @@ build_info: engine: "docker|native" output: "target/nextflow/pear" executable: "target/nextflow/pear/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/pear/main.nf b/target/nextflow/pear/main.nf index 0a0f8086..1835923f 100644 --- a/target/nextflow/pear/main.nf +++ b/target/nextflow/pear/main.nf @@ -1,6 +1,6 @@ // pear main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -763,8 +763,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -786,7 +789,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -824,8 +840,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1601,8 +1620,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2629,30 +2648,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2723,7 +2743,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -3232,15 +3252,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/pear", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/qualimap/qualimap_rnaseq/.config.vsh.yaml b/target/nextflow/qualimap/qualimap_rnaseq/.config.vsh.yaml index c5ee581d..2801d0e2 100644 --- a/target/nextflow/qualimap/qualimap_rnaseq/.config.vsh.yaml +++ b/target/nextflow/qualimap/qualimap_rnaseq/.config.vsh.yaml @@ -263,15 +263,15 @@ build_info: engine: "docker|native" output: "target/nextflow/qualimap/qualimap_rnaseq" executable: "target/nextflow/qualimap/qualimap_rnaseq/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/qualimap/qualimap_rnaseq/main.nf b/target/nextflow/qualimap/qualimap_rnaseq/main.nf index 286420d2..fda53697 100644 --- a/target/nextflow/qualimap/qualimap_rnaseq/main.nf +++ b/target/nextflow/qualimap/qualimap_rnaseq/main.nf @@ -1,6 +1,6 @@ // qualimap_rnaseq main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -763,8 +763,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -786,7 +789,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -824,8 +840,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1601,8 +1620,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2629,30 +2648,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2723,7 +2743,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -3102,15 +3122,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/qualimap/qualimap_rnaseq", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/rsem/rsem_prepare_reference/.config.vsh.yaml b/target/nextflow/rsem/rsem_prepare_reference/.config.vsh.yaml new file mode 100644 index 00000000..e3e16047 --- /dev/null +++ b/target/nextflow/rsem/rsem_prepare_reference/.config.vsh.yaml @@ -0,0 +1,442 @@ +name: "rsem_prepare_reference" +namespace: "rsem" +version: "main" +authors: +- name: "Sai Nirmayi Yasa" + roles: + - "author" + - "maintainer" + info: + links: + email: "nirmayi@data-intuitive.com" + github: "sainirmayi" + linkedin: "sai-nirmayi-yasa" + organizations: + - name: "Data Intuitive" + href: "https://www.data-intuitive.com" + role: "Junior Bioinformatics Researcher" +argument_groups: +- name: "Inputs" + arguments: + - type: "file" + name: "--reference_fasta_files" + description: "Semi-colon separated list of Multi-FASTA formatted files OR a directory\ + \ name. If a directory name is specified, RSEM will read all files with suffix\ + \ \".fa\" or \".fasta\" in this directory. The files should contain either the\ + \ sequences of transcripts or an entire genome, depending on whether the '--gtf'\ + \ option is used.\n" + info: null + example: + - "read1.fasta" + must_exist: true + create_parent: true + required: true + direction: "input" + multiple: true + multiple_sep: ";" + - type: "string" + name: "--reference_name" + description: "The name of the reference used. RSEM will generate several reference-related\ + \ files that are prefixed by this name. This name can contain path information\ + \ (e.g. '/ref/mm9').\n" + info: null + example: + - "/ref/mm9" + required: true + direction: "input" + multiple: false + multiple_sep: ";" +- name: "Outputs" + arguments: + - type: "file" + name: "--output" + description: "Directory containing reference files generated by RSEM." + info: null + must_exist: true + create_parent: true + required: true + direction: "output" + multiple: false + multiple_sep: ";" +- name: "Other options" + arguments: + - type: "file" + name: "--gtf" + description: "Assume that 'reference_fasta_files' contains the sequence of a genome,\ + \ and extract transcript reference sequences using the gene annotations specified\ + \ in the GTF file. If this and '--gff3' options are not provided, RSEM will\ + \ assume 'reference_fasta_files' contains the reference transcripts. In this\ + \ case, RSEM assumes that name of each sequence in the Multi-FASTA files is\ + \ its transcript_id." + info: null + example: + - "annotations.gtf" + must_exist: true + create_parent: true + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "file" + name: "--gff3" + description: "GFF3 annotation file. Converted to GTF format with the file name\ + \ 'reference_name.gtf'. Please make sure that 'reference_name.gtf' does not\ + \ exist." + info: null + example: + - "annotations.gff" + must_exist: true + create_parent: true + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "string" + name: "--gff3_rna_patterns" + description: "List of transcript categories (separated by semi-colon). Only transcripts\ + \ that match the string will be extracted." + info: null + example: + - "mRNA;rRNA" + required: false + direction: "input" + multiple: true + multiple_sep: ";" + - type: "boolean_true" + name: "--gff3_genes_as_transcripts" + description: "This option is designed for untypical organisms, such as viruses,\ + \ whose GFF3 files only contain genes. RSEM will assume each gene as a unique\ + \ transcript when it converts the GFF3 file into GTF format." + info: null + direction: "input" + - type: "string" + name: "--trusted_sources" + description: "List of trusted sources (separated by semi-colon). Only transcripts\ + \ coming from these sources will be extracted. If this option is off, all sources\ + \ are accepted." + info: null + example: + - "ENSEMBL;HAVANA" + required: false + direction: "input" + multiple: true + multiple_sep: ";" + - type: "file" + name: "--transcript_to_gene_map" + description: "Use information from this file to map from transcript (isoform)\ + \ ids to gene ids. Each line of this file should be of the form: \n gene_id\ + \ transcript_id\nwith the two fields separated by a tab character.\nIf you are\ + \ using a GTF file for the \"UCSC Genes\" gene set from the UCSC Genome Browser,\ + \ then the \"knownIsoforms.txt\" file (obtained from the \"Downloads\" section\ + \ of the UCSC Genome Browser site) is of this format. \nIf this option is off,\ + \ then the mapping of isoforms to genes depends on whether the '--gtf' option\ + \ is specified. If '--gtf' is specified, then RSEM uses the \"gene_id\" and\ + \ \"transcript_id\" attributes in the GTF file. Otherwise, RSEM assumes that\ + \ each sequence in the reference sequence files is a separate gene.\n" + info: null + example: + - "isoforms.txt" + must_exist: true + create_parent: true + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "file" + name: "--allele_to_gene_map" + description: "Use information from to provide gene_id and transcript_id\ + \ information for each allele-specific transcript. Each line of should\ + \ be of the form:\n gene_id transcript_id allele_id\nwith the fields separated\ + \ by a tab character.\nThis option is designed for quantifying allele-specific\ + \ expression. It is only valid if '--gtf' option is not specified. allele_id\ + \ should be the sequence names presented in the Multi-FASTA-formatted files.\n" + info: null + must_exist: true + create_parent: true + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "boolean_true" + name: "--polyA" + description: "Add poly(A) tails to the end of all reference isoforms. The length\ + \ of poly(A) tail added is specified by '--polyA-length' option. STAR aligner\ + \ users may not want to use this option." + info: null + direction: "input" + - type: "integer" + name: "--polyA_length" + description: "The length of the poly(A) tails to be added." + info: null + example: + - 125 + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "file" + name: "--no_polyA_subset" + description: "Only meaningful if '--polyA' is specified. Do not add poly(A) tails\ + \ to those transcripts listed in this file containing a list of transcript_ids." + info: null + example: + - "transcript_ids.txt" + must_exist: true + create_parent: true + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "boolean_true" + name: "--bowtie" + description: "Build Bowtie indices." + info: null + direction: "input" + - type: "boolean_true" + name: "--bowtie2" + description: "Build Bowtie 2 indices." + info: null + direction: "input" + - type: "boolean_true" + name: "--star" + description: "Build STAR indices." + info: null + direction: "input" + - type: "integer" + name: "--star_sjdboverhang" + description: "Length of the genomic sequence around annotated junction. It is\ + \ only used for STAR to build splice junctions database and not needed for Bowtie\ + \ or Bowtie2. It will be passed as the --sjdbOverhang option to STAR. According\ + \ to STAR's manual, its ideal value is max(ReadLength)-1, e.g. for 2x101 paired-end\ + \ reads, the ideal value is 101-1=100. In most cases, the default value of 100\ + \ will work as well as the ideal value. (Default is 100)" + info: null + example: + - 100 + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "boolean_true" + name: "--hisat2_hca" + description: "Build HISAT2 indices on the transcriptome according to Human Cell\ + \ Atlas (HCA) SMART-Seq2 pipeline." + info: null + direction: "input" + - type: "boolean_true" + name: "--quiet" + alternatives: + - "-q" + description: "Suppress the output of logging information." + info: null + direction: "input" +- name: "Prior-enhanced RSEM options" + arguments: + - type: "boolean_true" + name: "--prep_pRSEM" + description: "A Boolean indicating whether to prepare reference files for pRSEM,\ + \ including building Bowtie indices for a genome and selecting training set\ + \ isoforms. The index files will be used for aligning ChIP-seq reads in prior-enhanced\ + \ RSEM and the training set isoforms will be used for learning prior. A path\ + \ to Bowtie executables and a mappability file in bigWig format are required\ + \ when this option is on. Currently, Bowtie2 is not supported for prior-enhanced\ + \ RSEM." + info: null + direction: "input" + - type: "file" + name: "--mappability_bigwig_file" + description: "Full path to a whole-genome mappability file in bigWig format. This\ + \ file is required for running prior-enhanced RSEM. It is used for selecting\ + \ a training set of isoforms for prior-learning. This file can be either downloaded\ + \ from UCSC Genome Browser or generated by GEM (Derrien et al., 2012, PLoS One)." + info: null + must_exist: true + create_parent: true + required: false + direction: "input" + multiple: false + multiple_sep: ";" +resources: +- type: "bash_script" + path: "script.sh" + is_executable: true +description: "RSEM is a software package for estimating gene and isoform expression\ + \ levels from RNA-Seq data. This component prepares transcript references for RSEM.\n" +test_resources: +- type: "bash_script" + path: "test.sh" + is_executable: true +info: null +status: "enabled" +requirements: + commands: + - "ps" +keywords: +- "Transcriptome" +- "Index" +license: "GPL-3.0" +references: + doi: + - "10.1186/1471-2105-12-323" +links: + repository: "https://github.com/deweylab/RSEM" + homepage: "http://deweylab.github.io/RSEM" + documentation: "https://deweylab.github.io/RSEM/rsem-prepare-reference.html" +runners: +- type: "executable" + id: "executable" + docker_setup_strategy: "ifneedbepullelsecachedbuild" +- type: "nextflow" + id: "nextflow" + directives: + tag: "$id" + auto: + simplifyInput: true + simplifyOutput: false + transcript: false + publish: false + config: + labels: + mem1gb: "memory = 1000000000.B" + mem2gb: "memory = 2000000000.B" + mem5gb: "memory = 5000000000.B" + mem10gb: "memory = 10000000000.B" + mem20gb: "memory = 20000000000.B" + mem50gb: "memory = 50000000000.B" + mem100gb: "memory = 100000000000.B" + mem200gb: "memory = 200000000000.B" + mem500gb: "memory = 500000000000.B" + mem1tb: "memory = 1000000000000.B" + mem2tb: "memory = 2000000000000.B" + mem5tb: "memory = 5000000000000.B" + mem10tb: "memory = 10000000000000.B" + mem20tb: "memory = 20000000000000.B" + mem50tb: "memory = 50000000000000.B" + mem100tb: "memory = 100000000000000.B" + mem200tb: "memory = 200000000000000.B" + mem500tb: "memory = 500000000000000.B" + mem1gib: "memory = 1073741824.B" + mem2gib: "memory = 2147483648.B" + mem4gib: "memory = 4294967296.B" + mem8gib: "memory = 8589934592.B" + mem16gib: "memory = 17179869184.B" + mem32gib: "memory = 34359738368.B" + mem64gib: "memory = 68719476736.B" + mem128gib: "memory = 137438953472.B" + mem256gib: "memory = 274877906944.B" + mem512gib: "memory = 549755813888.B" + mem1tib: "memory = 1099511627776.B" + mem2tib: "memory = 2199023255552.B" + mem4tib: "memory = 4398046511104.B" + mem8tib: "memory = 8796093022208.B" + mem16tib: "memory = 17592186044416.B" + mem32tib: "memory = 35184372088832.B" + mem64tib: "memory = 70368744177664.B" + mem128tib: "memory = 140737488355328.B" + mem256tib: "memory = 281474976710656.B" + mem512tib: "memory = 562949953421312.B" + cpu1: "cpus = 1" + cpu2: "cpus = 2" + cpu5: "cpus = 5" + cpu10: "cpus = 10" + cpu20: "cpus = 20" + cpu50: "cpus = 50" + cpu100: "cpus = 100" + cpu200: "cpus = 200" + cpu500: "cpus = 500" + cpu1000: "cpus = 1000" + debug: false + container: "docker" +engines: +- type: "docker" + id: "docker" + image: "ubuntu:22.04" + target_registry: "images.viash-hub.com" + target_tag: "main" + namespace_separator: "/" + setup: + - type: "apt" + packages: + - "build-essential" + - "gcc" + - "g++" + - "make" + - "wget" + - "zlib1g-dev" + - "unzip xxd" + - "perl" + - "r-base" + - "bowtie2" + - "pip" + - "git" + interactive: false + - type: "python" + user: false + packages: + - "bowtie" + upgrade: true + - type: "docker" + run: + - "ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone\ + \ && \\\ncd /tmp && \\\nwget --no-check-certificate https://github.com/alexdobin/STAR/archive/refs/tags/${STAR_VERSION}.zip\ + \ && \\\nunzip ${STAR_VERSION}.zip && \\\ncd STAR-${STAR_VERSION}/source &&\ + \ \\\nmake STARstatic CXXFLAGS_SIMD=-std=c++11 && \\\ncp STAR /usr/local/bin\ + \ && \\\ncd /tmp && \\\nwget --no-check-certificate https://github.com/deweylab/RSEM/archive/refs/tags/v${RSEM_VERSION}.zip\ + \ && \\\nunzip v${RSEM_VERSION}.zip && \\\ncd RSEM-${RSEM_VERSION} && \\\nmake\ + \ && \\\nmake install && \\\ncd /tmp && \\\nwget --no-check-certificate -O bowtie-${BOWTIE_VERSION}-linux-x86_64.zip\ + \ https://sourceforge.net/projects/bowtie-bio/files/bowtie/${BOWTIE_VERSION}/bowtie-${BOWTIE_VERSION}-linux-x86_64.zip/download\ + \ && \\\nunzip bowtie-${BOWTIE_VERSION}-linux-x86_64.zip && \\\ncp bowtie-${BOWTIE_VERSION}-linux-x86_64/bowtie*\ + \ /usr/local/bin && \\\ncd /tmp && \\\ngit clone https://github.com/DaehwanKimLab/hisat2.git\ + \ /tmp/hisat2 && \\\ncd /tmp/hisat2 && \\\nmake && \\\ncp -r hisat2* /usr/local/bin\ + \ && \\\ncd && \\\nrm -rf /tmp/STAR-${STAR_VERSION} /tmp/${STAR_VERSION}.zip\ + \ /tmp/bowtie-${BOWTIE_VERSION}-linux-x86_64 /tmp/hisat2 && \\\napt-get --purge\ + \ autoremove -y ${PACKAGES} && \\\napt-get clean \n" + env: + - "STAR_VERSION=2.7.11b" + - "RSEM_VERSION=1.3.3" + - "BOWTIE_VERSION=1.3.1" + - "TZ=Europe/Brussels" + - type: "docker" + run: + - "echo \"RSEM: `rsem-calculate-expression --version | sed -e 's/Current version:\ + \ RSEM v//g'`\" > /var/software_versions.txt && \\\necho \"STAR: `STAR --version`\"\ + \ >> /var/software_versions.txt && \\\necho \"bowtie2: `bowtie2 --version |\ + \ grep -oP '\\d+\\.\\d+\\.\\d+'`\" >> /var/software_versions.txt && \\\necho\ + \ \"bowtie: `bowtie --version | grep -oP 'bowtie-align-s version \\K\\d+\\.\\\ + d+\\.\\d+'`\" >> /var/software_versions.txt && \\\necho \"HISAT2: `hisat2 --version\ + \ | grep -oP 'hisat2-align-s version \\K\\d+\\.\\d+\\.\\d+'`\" >> /var/software_versions.txt\n" + entrypoint: [] + cmd: null +- type: "native" + id: "native" +build_info: + config: "src/rsem/rsem_prepare_reference/config.vsh.yaml" + runner: "nextflow" + engine: "docker|native" + output: "target/nextflow/rsem/rsem_prepare_reference" + executable: "target/nextflow/rsem/rsem_prepare_reference/main.nf" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" + git_remote: "https://github.com/viash-hub/biobox" +package_config: + name: "biobox" + version: "main" + description: "A collection of bioinformatics tools for working with sequence data.\n" + info: null + viash_version: "0.9.0-RC7" + source: "src" + target: "target" + config_mods: + - ".requirements.commands := ['ps']\n" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'main'" + keywords: + - "bioinformatics" + - "modules" + - "sequencing" + license: "MIT" + organization: "vsh" + links: + repository: "https://github.com/viash-hub/biobox" + issue_tracker: "https://github.com/viash-hub/biobox/issues" diff --git a/target/nextflow/rsem/rsem_prepare_reference/main.nf b/target/nextflow/rsem/rsem_prepare_reference/main.nf new file mode 100644 index 00000000..558c50fb --- /dev/null +++ b/target/nextflow/rsem/rsem_prepare_reference/main.nf @@ -0,0 +1,3842 @@ +// rsem_prepare_reference main +// +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a +// derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from +// Data Intuitive. +// +// The component may contain files which fall under a different license. The +// authors of this component should specify the license in the header of such +// files, or include a separate license file detailing the licenses of all included +// files. +// +// Component authors: +// * Sai Nirmayi Yasa (author, maintainer) + +//////////////////////////// +// VDSL3 helper functions // +//////////////////////////// + +// helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_checkArgumentType.nf' +class UnexpectedArgumentTypeException extends Exception { + String errorIdentifier + String stage + String plainName + String expectedClass + String foundClass + + // ${key ? " in module '$key'" : ""}${id ? " id '$id'" : ""} + UnexpectedArgumentTypeException(String errorIdentifier, String stage, String plainName, String expectedClass, String foundClass) { + super("Error${errorIdentifier ? " $errorIdentifier" : ""}:${stage ? " $stage" : "" } argument '${plainName}' has the wrong type. " + + "Expected type: ${expectedClass}. Found type: ${foundClass}") + this.errorIdentifier = errorIdentifier + this.stage = stage + this.plainName = plainName + this.expectedClass = expectedClass + this.foundClass = foundClass + } +} + +/** + * Checks if the given value is of the expected type. If not, an exception is thrown. + * + * @param stage The stage of the argument (input or output) + * @param par The parameter definition + * @param value The value to check + * @param errorIdentifier The identifier to use in the error message + * @return The value, if it is of the expected type + * @throws UnexpectedArgumentTypeException If the value is not of the expected type +*/ +def _checkArgumentType(String stage, Map par, Object value, String errorIdentifier) { + // expectedClass will only be != null if value is not of the expected type + def expectedClass = null + def foundClass = null + + // todo: split if need be + + if (!par.required && value == null) { + expectedClass = null + } else if (par.multiple) { + if (value !instanceof Collection) { + value = [value] + } + + // split strings + value = value.collectMany{ val -> + if (val instanceof String) { + // collect() to ensure that the result is a List and not simply an array + val.split(par.multiple_sep).collect() + } else { + [val] + } + } + + // process globs + if (par.type == "file" && par.direction == "input") { + value = value.collect{ it instanceof String ? file(it, hidden: true) : it }.flatten() + } + + // check types of elements in list + try { + value = value.collect { listVal -> + _checkArgumentType(stage, par + [multiple: false], listVal, errorIdentifier) + } + } catch (UnexpectedArgumentTypeException e) { + expectedClass = "List[${e.expectedClass}]" + foundClass = "List[${e.foundClass}]" + } + } else if (par.type == "string") { + // cast to string if need be + if (value instanceof GString) { + value = value.toString() + } + expectedClass = value instanceof String ? null : "String" + } else if (par.type == "integer") { + // cast to integer if need be + if (value instanceof String) { + try { + value = value.toInteger() + } catch (NumberFormatException e) { + // do nothing + } + } + if (value instanceof java.math.BigInteger) { + value = value.intValue() + } + expectedClass = value instanceof Integer ? null : "Integer" + } else if (par.type == "long") { + // cast to long if need be + if (value instanceof String) { + try { + value = value.toLong() + } catch (NumberFormatException e) { + // do nothing + } + } + if (value instanceof Integer) { + value = value.toLong() + } + expectedClass = value instanceof Long ? null : "Long" + } else if (par.type == "double") { + // cast to double if need be + if (value instanceof String) { + try { + value = value.toDouble() + } catch (NumberFormatException e) { + // do nothing + } + } + if (value instanceof java.math.BigDecimal) { + value = value.doubleValue() + } + if (value instanceof Float) { + value = value.toDouble() + } + expectedClass = value instanceof Double ? null : "Double" + } else if (par.type == "boolean" | par.type == "boolean_true" | par.type == "boolean_false") { + // cast to boolean if need be + if (value instanceof String) { + def valueLower = value.toLowerCase() + if (valueLower == "true") { + value = true + } else if (valueLower == "false") { + value = false + } + } + expectedClass = value instanceof Boolean ? null : "Boolean" + } else if (par.type == "file" && (par.direction == "input" || stage == "output")) { + // cast to path if need be + if (value instanceof String) { + value = file(value, hidden: true) + } + if (value instanceof File) { + value = value.toPath() + } + expectedClass = value instanceof Path ? null : "Path" + } else if (par.type == "file" && stage == "input" && par.direction == "output") { + // cast to string if need be + if (value instanceof GString) { + value = value.toString() + } + expectedClass = value instanceof String ? null : "String" + } else { + // didn't find a match for par.type + expectedClass = par.type + } + + if (expectedClass != null) { + if (foundClass == null) { + foundClass = value.getClass().getName() + } + throw new UnexpectedArgumentTypeException(errorIdentifier, stage, par.plainName, expectedClass, foundClass) + } + + return value +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_processInputValues.nf' +Map _processInputValues(Map inputs, Map config, String id, String key) { + if (!workflow.stubRun) { + config.allArguments.each { arg -> + if (arg.required) { + assert inputs.containsKey(arg.plainName) && inputs.get(arg.plainName) != null : + "Error in module '${key}' id '${id}': required input argument '${arg.plainName}' is missing" + } + } + + inputs = inputs.collectEntries { name, value -> + def par = config.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") } + assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid input argument" + + value = _checkArgumentType("input", par, value, "in module '$key' id '$id'") + + [ name, value ] + } + } + return inputs +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_processOutputValues.nf' +Map _processOutputValues(Map outputs, Map config, String id, String key) { + if (!workflow.stubRun) { + config.allArguments.each { arg -> + if (arg.direction == "output" && arg.required) { + assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : + "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing" + } + } + + outputs = outputs.collectEntries { name, value -> + def par = config.allArguments.find { it.plainName == name && it.direction == "output" } + assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid output argument" + + value = _checkArgumentType("output", par, value, "in module '$key' id '$id'") + + [ name, value ] + } + } + return outputs +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/IDChecker.nf' +class IDChecker { + final def items = [] as Set + + @groovy.transform.WithWriteLock + boolean observe(String item) { + if (items.contains(item)) { + return false + } else { + items << item + return true + } + } + + @groovy.transform.WithReadLock + boolean contains(String item) { + return items.contains(item) + } + + @groovy.transform.WithReadLock + Set getItems() { + return items.clone() + } +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_checkUniqueIds.nf' + +/** + * Check if the ids are unique across parameter sets + * + * @param parameterSets a list of parameter sets. + */ +private void _checkUniqueIds(List>> parameterSets) { + def ppIds = parameterSets.collect{it[0]} + assert ppIds.size() == ppIds.unique().size() : "All argument sets should have unique ids. Detected ids: $ppIds" +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_getChild.nf' + +// helper functions for reading params from file // +def _getChild(parent, child) { + if (child.contains("://") || java.nio.file.Paths.get(child).isAbsolute()) { + child + } else { + def parentAbsolute = java.nio.file.Paths.get(parent).toAbsolutePath().toString() + parentAbsolute.replaceAll('/[^/]*$', "/") + child + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_parseParamList.nf' +/** + * Figure out the param list format based on the file extension + * + * @param param_list A String containing the path to the parameter list file. + * + * @return A String containing the format of the parameter list file. + */ +def _paramListGuessFormat(param_list) { + if (param_list !instanceof String) { + "asis" + } else if (param_list.endsWith(".csv")) { + "csv" + } else if (param_list.endsWith(".json") || param_list.endsWith(".jsn")) { + "json" + } else if (param_list.endsWith(".yaml") || param_list.endsWith(".yml")) { + "yaml" + } else { + "yaml_blob" + } +} + + +/** + * Read the param list + * + * @param param_list One of the following: + * - A String containing the path to the parameter list file (csv, json or yaml), + * - A yaml blob of a list of maps (yaml_blob), + * - Or a groovy list of maps (asis). + * @param config A Map of the Viash configuration. + * + * @return A List of Maps containing the parameters. + */ +def _parseParamList(param_list, Map config) { + // first determine format by extension + def paramListFormat = _paramListGuessFormat(param_list) + + def paramListPath = (paramListFormat != "asis" && paramListFormat != "yaml_blob") ? + file(param_list, hidden: true) : + null + + // get the correct parser function for the detected params_list format + def paramSets = [] + if (paramListFormat == "asis") { + paramSets = param_list + } else if (paramListFormat == "yaml_blob") { + paramSets = readYamlBlob(param_list) + } else if (paramListFormat == "yaml") { + paramSets = readYaml(paramListPath) + } else if (paramListFormat == "json") { + paramSets = readJson(paramListPath) + } else if (paramListFormat == "csv") { + paramSets = readCsv(paramListPath) + } else { + error "Format of provided --param_list not recognised.\n" + + "Found: '$paramListFormat'.\n" + + "Expected: a csv file, a json file, a yaml file,\n" + + "a yaml blob or a groovy list of maps." + } + + // data checks + assert paramSets instanceof List: "--param_list should contain a list of maps" + for (value in paramSets) { + assert value instanceof Map: "--param_list should contain a list of maps" + } + + // id is argument + def idIsArgument = config.allArguments.any{it.plainName == "id"} + + // Reformat from List to List> by adding the ID as first element of a Tuple2 + paramSets = paramSets.collect({ data -> + def id = data.id + if (!idIsArgument) { + data = data.findAll{k, v -> k != "id"} + } + [id, data] + }) + + // Split parameters with 'multiple: true' + paramSets = paramSets.collect({ id, data -> + data = _splitParams(data, config) + [id, data] + }) + + // The paths of input files inside a param_list file may have been specified relatively to the + // location of the param_list file. These paths must be made absolute. + if (paramListPath) { + paramSets = paramSets.collect({ id, data -> + def new_data = data.collectEntries{ parName, parValue -> + def par = config.allArguments.find{it.plainName == parName} + if (par && par.type == "file" && par.direction == "input") { + if (parValue instanceof Collection) { + parValue = parValue.collectMany{path -> + def x = _resolveSiblingIfNotAbsolute(path, paramListPath) + x instanceof Collection ? x : [x] + } + } else { + parValue = _resolveSiblingIfNotAbsolute(parValue, paramListPath) + } + } + [parName, parValue] + } + [id, new_data] + }) + } + + return paramSets +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_splitParams.nf' +/** + * Split parameters for arguments that accept multiple values using their separator + * + * @param paramList A Map containing parameters to split. + * @param config A Map of the Viash configuration. This Map can be generated from the config file + * using the readConfig() function. + * + * @return A Map of parameters where the parameter values have been split into a list using + * their seperator. + */ +Map _splitParams(Map parValues, Map config){ + def parsedParamValues = parValues.collectEntries { parName, parValue -> + def parameterSettings = config.allArguments.find({it.plainName == parName}) + + if (!parameterSettings) { + // if argument is not found, do not alter + return [parName, parValue] + } + if (parameterSettings.multiple) { // Check if parameter can accept multiple values + if (parValue instanceof Collection) { + parValue = parValue.collect{it instanceof String ? it.split(parameterSettings.multiple_sep) : it } + } else if (parValue instanceof String) { + parValue = parValue.split(parameterSettings.multiple_sep) + } else if (parValue == null) { + parValue = [] + } else { + parValue = [ parValue ] + } + parValue = parValue.flatten() + } + // For all parameters check if multiple values are only passed for + // arguments that allow it. Quietly simplify lists of length 1. + if (!parameterSettings.multiple && parValue instanceof Collection) { + assert parValue.size() == 1 : + "Error: argument ${parName} has too many values.\n" + + " Expected amount: 1. Found: ${parValue.size()}" + parValue = parValue[0] + } + [parName, parValue] + } + return parsedParamValues +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/channelFromParams.nf' +/** + * Parse nextflow parameters based on settings defined in a viash config. + * Return a list of parameter sets, each parameter set corresponding to + * an event in a nextflow channel. The output from this function can be used + * with Channel.fromList to create a nextflow channel with Vdsl3 formatted + * events. + * + * This function performs: + * - A filtering of the params which can be found in the config file. + * - Process the params_list argument which allows a user to to initialise + * a Vsdl3 channel with multiple parameter sets. Possible formats are + * csv, json, yaml, or simply a yaml_blob. A csv should have column names + * which correspond to the different arguments of this pipeline. A json or a yaml + * file should be a list of maps, each of which has keys corresponding to the + * arguments of the pipeline. A yaml blob can also be passed directly as a parameter. + * When passing a csv, json or yaml, relative path names are relativized to the + * location of the parameter file. + * - Combine the parameter sets into a vdsl3 Channel. + * + * @param params Input parameters. Can optionaly contain a 'param_list' key that + * provides a list of arguments that can be split up into multiple events + * in the output channel possible formats of param_lists are: a csv file, + * json file, a yaml file or a yaml blob. Each parameters set (event) must + * have a unique ID. + * @param config A Map of the Viash configuration. This Map can be generated from the config file + * using the readConfig() function. + * + * @return A list of parameters with the first element of the event being + * the event ID and the second element containing a map of the parsed parameters. + */ + +private List>> _paramsToParamSets(Map params, Map config){ + // todo: fetch key from run args + def key_ = config.name + + /* parse regular parameters (not in param_list) */ + /*************************************************/ + def globalParams = config.allArguments + .findAll { params.containsKey(it.plainName) } + .collectEntries { [ it.plainName, params[it.plainName] ] } + def globalID = params.get("id", null) + + /* process params_list arguments */ + /*********************************/ + def paramList = params.containsKey("param_list") && params.param_list != null ? + params.param_list : [] + // if (paramList instanceof String) { + // paramList = [paramList] + // } + // def paramSets = paramList.collectMany{ _parseParamList(it, config) } + // TODO: be able to process param_list when it is a list of strings + def paramSets = _parseParamList(paramList, config) + if (paramSets.isEmpty()) { + paramSets = [[null, [:]]] + } + + /* combine arguments into channel */ + /**********************************/ + def processedParams = paramSets.indexed().collect{ index, tup -> + // Process ID + def id = tup[0] ?: globalID + + if (workflow.stubRun && !id) { + // if stub run, explicitly add an id if missing + id = "stub${index}" + } + assert id != null: "Each parameter set should have at least an 'id'" + + // Process params + def parValues = globalParams + tup[1] + // // Remove parameters which are null, if the default is also null + // parValues = parValues.collectEntries{paramName, paramValue -> + // parameterSettings = config.functionality.allArguments.find({it.plainName == paramName}) + // if ( paramValue != null || parameterSettings.get("default", null) != null ) { + // [paramName, paramValue] + // } + // } + parValues = parValues.collectEntries { name, value -> + def par = config.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") } + assert par != null : "Error in module '${key_}' id '${id}': '${name}' is not a valid input argument" + + if (par == null) { + return [:] + } + value = _checkArgumentType("input", par, value, "in module '$key_' id '$id'") + + [ name, value ] + } + + [id, parValues] + } + + // Check if ids (first element of each list) is unique + _checkUniqueIds(processedParams) + return processedParams +} + +/** + * Parse nextflow parameters based on settings defined in a viash config + * and return a nextflow channel. + * + * @param params Input parameters. Can optionaly contain a 'param_list' key that + * provides a list of arguments that can be split up into multiple events + * in the output channel possible formats of param_lists are: a csv file, + * json file, a yaml file or a yaml blob. Each parameters set (event) must + * have a unique ID. + * @param config A Map of the Viash configuration. This Map can be generated from the config file + * using the readConfig() function. + * + * @return A nextflow Channel with events. Events are formatted as a tuple that contains + * first contains the ID of the event and as second element holds a parameter map. + * + * + */ +def channelFromParams(Map params, Map config) { + def processedParams = _paramsToParamSets(params, config) + return Channel.fromList(processedParams) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/checkUniqueIds.nf' +def checkUniqueIds(Map args) { + def stopOnError = args.stopOnError == null ? args.stopOnError : true + + def idChecker = new IDChecker() + + return filter { tup -> + if (!idChecker.observe(tup[0])) { + if (stopOnError) { + error "Duplicate id: ${tup[0]}" + } else { + log.warn "Duplicate id: ${tup[0]}, removing duplicate entry" + return false + } + } + return true + } +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/preprocessInputs.nf' +// This helper file will be deprecated soon +preprocessInputsDeprecationWarningPrinted = false + +def preprocessInputsDeprecationWarning() { + if (!preprocessInputsDeprecationWarningPrinted) { + preprocessInputsDeprecationWarningPrinted = true + System.err.println("Warning: preprocessInputs() is deprecated and will be removed in Viash 0.9.0.") + } +} + +/** + * Generate a nextflow Workflow that allows processing a channel of + * Vdsl3 formatted events and apply a Viash config to them: + * - Gather default parameters from the Viash config and make + * sure that they are correctly formatted (see applyConfig method). + * - Format the input parameters (also using the applyConfig method). + * - Apply the default parameter to the input parameters. + * - Do some assertions: + * ~ Check if the event IDs in the channel are unique. + * + * The events in the channel are formatted as tuples, with the + * first element of the tuples being a unique id of the parameter set, + * and the second element containg the the parameters themselves. + * Optional extra elements of the tuples will be passed to the output as is. + * + * @param args A map that must contain a 'config' key that points + * to a parsed config (see readConfig()). Optionally, a + * 'key' key can be provided which can be used to create a unique + * name for the workflow process. + * + * @return A workflow that allows processing a channel of Vdsl3 formatted events + * and apply a Viash config to them. + */ +def preprocessInputs(Map args) { + preprocessInputsDeprecationWarning() + + def config = args.config + assert config instanceof Map : + "Error in preprocessInputs: config must be a map. " + + "Expected class: Map. Found: config.getClass() is ${config.getClass()}" + def key_ = args.key ?: config.name + + // Get different parameter types (used throughout this function) + def defaultArgs = config.allArguments + .findAll { it.containsKey("default") } + .collectEntries { [ it.plainName, it.default ] } + + map { tup -> + def id = tup[0] + def data = tup[1] + def passthrough = tup.drop(2) + + def new_data = (defaultArgs + data).collectEntries { name, value -> + def par = config.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") } + + if (par != null) { + value = _checkArgumentType("input", par, value, "in module '$key_' id '$id'") + } + + [ name, value ] + } + + [ id, new_data ] + passthrough + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/runComponents.nf' +/** + * Run a list of components on a stream of data. + * + * @param components: list of Viash VDSL3 modules to run + * @param fromState: a closure, a map or a list of keys to extract from the input data. + * If a closure, it will be called with the id, the data and the component config. + * @param toState: a closure, a map or a list of keys to extract from the output data + * If a closure, it will be called with the id, the output data, the old state and the component config. + * @param filter: filter function to apply to the input. + * It will be called with the id, the data and the component config. + * @param id: id to use for the output data + * If a closure, it will be called with the id, the data and the component config. + * @param auto: auto options to pass to the components + * + * @return: a workflow that runs the components + **/ +def runComponents(Map args) { + log.warn("runComponents is deprecated, use runEach instead") + assert args.components: "runComponents should be passed a list of components to run" + + def components_ = args.components + if (components_ !instanceof List) { + components_ = [ components_ ] + } + assert components_.size() > 0: "pass at least one component to runComponents" + + def fromState_ = args.fromState + def toState_ = args.toState + def filter_ = args.filter + def id_ = args.id + + workflow runComponentsWf { + take: input_ch + main: + + // generate one channel per method + out_chs = components_.collect{ comp_ -> + def comp_config = comp_.config + + def filter_ch = filter_ + ? input_ch | filter{tup -> + filter_(tup[0], tup[1], comp_config) + } + : input_ch + def id_ch = id_ + ? filter_ch | map{tup -> + // def new_id = id_(tup[0], tup[1], comp_config) + def new_id = tup[0] + if (id_ instanceof String) { + new_id = id_ + } else if (id_ instanceof Closure) { + new_id = id_(new_id, tup[1], comp_config) + } + [new_id] + tup.drop(1) + } + : filter_ch + def data_ch = id_ch | map{tup -> + def new_data = tup[1] + if (fromState_ instanceof Map) { + new_data = fromState_.collectEntries{ key0, key1 -> + [key0, new_data[key1]] + } + } else if (fromState_ instanceof List) { + new_data = fromState_.collectEntries{ key -> + [key, new_data[key]] + } + } else if (fromState_ instanceof Closure) { + new_data = fromState_(tup[0], new_data, comp_config) + } + tup.take(1) + [new_data] + tup.drop(1) + } + def out_ch = data_ch + | comp_.run( + auto: (args.auto ?: [:]) + [simplifyInput: false, simplifyOutput: false] + ) + def post_ch = toState_ + ? out_ch | map{tup -> + def output = tup[1] + def old_state = tup[2] + def new_state = null + if (toState_ instanceof Map) { + new_state = old_state + toState_.collectEntries{ key0, key1 -> + [key0, output[key1]] + } + } else if (toState_ instanceof List) { + new_state = old_state + toState_.collectEntries{ key -> + [key, output[key]] + } + } else if (toState_ instanceof Closure) { + new_state = toState_(tup[0], output, old_state, comp_config) + } + [tup[0], new_state] + tup.drop(3) + } + : out_ch + + post_ch + } + + // mix all results + output_ch = + (out_chs.size == 1) + ? out_chs[0] + : out_chs[0].mix(*out_chs.drop(1)) + + emit: output_ch + } + + return runComponentsWf +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/runEach.nf' +/** + * Run a list of components on a stream of data. + * + * @param components: list of Viash VDSL3 modules to run + * @param fromState: a closure, a map or a list of keys to extract from the input data. + * If a closure, it will be called with the id, the data and the component itself. + * @param toState: a closure, a map or a list of keys to extract from the output data + * If a closure, it will be called with the id, the output data, the old state and the component itself. + * @param filter: filter function to apply to the input. + * It will be called with the id, the data and the component itself. + * @param id: id to use for the output data + * If a closure, it will be called with the id, the data and the component itself. + * @param auto: auto options to pass to the components + * + * @return: a workflow that runs the components + **/ +def runEach(Map args) { + assert args.components: "runEach should be passed a list of components to run" + + def components_ = args.components + if (components_ !instanceof List) { + components_ = [ components_ ] + } + assert components_.size() > 0: "pass at least one component to runEach" + + def fromState_ = args.fromState + def toState_ = args.toState + def filter_ = args.filter + def runIf_ = args.runIf + def id_ = args.id + + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + + workflow runEachWf { + take: input_ch + main: + + // generate one channel per method + out_chs = components_.collect{ comp_ -> + def filter_ch = filter_ + ? input_ch | filter{tup -> + filter_(tup[0], tup[1], comp_) + } + : input_ch + def id_ch = id_ + ? filter_ch | map{tup -> + def new_id = id_ + if (new_id instanceof Closure) { + new_id = new_id(tup[0], tup[1], comp_) + } + assert new_id instanceof String : "Error in runEach: id should be a String or a Closure that returns a String. Expected: id instanceof String. Found: ${new_id.getClass()}" + [new_id] + tup.drop(1) + } + : filter_ch + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> + def new_data = tup[1] + if (fromState_ instanceof Map) { + new_data = fromState_.collectEntries{ key0, key1 -> + [key0, new_data[key1]] + } + } else if (fromState_ instanceof List) { + new_data = fromState_.collectEntries{ key -> + [key, new_data[key]] + } + } else if (fromState_ instanceof Closure) { + new_data = fromState_(tup[0], new_data, comp_) + } + tup.take(1) + [new_data] + tup.drop(1) + } + def out_ch = data_ch + | comp_.run( + auto: (args.auto ?: [:]) + [simplifyInput: false, simplifyOutput: false] + ) + def post_ch = toState_ + ? out_ch | map{tup -> + def output = tup[1] + def old_state = tup[2] + def new_state = null + if (toState_ instanceof Map) { + new_state = old_state + toState_.collectEntries{ key0, key1 -> + [key0, output[key1]] + } + } else if (toState_ instanceof List) { + new_state = old_state + toState_.collectEntries{ key -> + [key, output[key]] + } + } else if (toState_ instanceof Closure) { + new_state = toState_(tup[0], output, old_state, comp_) + } + [tup[0], new_state] + tup.drop(3) + } + : out_ch + + def return_ch = post_ch + | concat(chPassthrough) + + return_ch + } + + // mix all results + output_ch = + (out_chs.size == 1) + ? out_chs[0] + : out_chs[0].mix(*out_chs.drop(1)) + + emit: output_ch + } + + return runEachWf +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/safeJoin.nf' +/** + * Join sourceChannel to targetChannel + * + * This function joins the sourceChannel to the targetChannel. + * However, each id in the targetChannel must be present in the + * sourceChannel. If _meta.join_id exists in the targetChannel, that is + * used as an id instead. If the id doesn't match any id in the sourceChannel, + * an error is thrown. + */ + +def safeJoin(targetChannel, sourceChannel, key) { + def sourceIDs = new IDChecker() + + def sourceCheck = sourceChannel + | map { tup -> + sourceIDs.observe(tup[0]) + tup + } + def targetCheck = targetChannel + | map { tup -> + def id = tup[0] + + if (!sourceIDs.contains(id)) { + error ( + "Error in module '${key}' when merging output with original state.\n" + + " Reason: output with id '${id}' could not be joined with source channel.\n" + + " If the IDs in the output channel differ from the input channel,\n" + + " please set `tup[1]._meta.join_id to the original ID.\n" + + " Original IDs in input channel: ['${sourceIDs.getItems().join("', '")}'].\n" + + " Unexpected ID in the output channel: '${id}'.\n" + + " Example input event: [\"id\", [input: file(...)]],\n" + + " Example output event: [\"newid\", [output: file(...), _meta: [join_id: \"id\"]]]" + ) + } + // TODO: add link to our documentation on how to fix this + + tup + } + + sourceCheck.cross(targetChannel) + | map{ left, right -> + right + left.drop(1) + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/config/_processArgument.nf' +def _processArgument(arg) { + arg.multiple = arg.multiple != null ? arg.multiple : false + arg.required = arg.required != null ? arg.required : false + arg.direction = arg.direction != null ? arg.direction : "input" + arg.multiple_sep = arg.multiple_sep != null ? arg.multiple_sep : ";" + arg.plainName = arg.name.replaceAll("^-*", "") + + if (arg.type == "file") { + arg.must_exist = arg.must_exist != null ? arg.must_exist : true + arg.create_parent = arg.create_parent != null ? arg.create_parent : true + } + + // add default values to output files which haven't already got a default + if (arg.type == "file" && arg.direction == "output" && arg.default == null) { + def mult = arg.multiple ? "_*" : "" + def extSearch = "" + if (arg.default != null) { + extSearch = arg.default + } else if (arg.example != null) { + extSearch = arg.example + } + if (extSearch instanceof List) { + extSearch = extSearch[0] + } + def extSearchResult = extSearch.find("\\.[^\\.]+\$") + def ext = extSearchResult != null ? extSearchResult : "" + arg.default = "\$id.\$key.${arg.plainName}${mult}${ext}" + if (arg.multiple) { + arg.default = [arg.default] + } + } + + if (!arg.multiple) { + if (arg.default != null && arg.default instanceof List) { + arg.default = arg.default[0] + } + if (arg.example != null && arg.example instanceof List) { + arg.example = arg.example[0] + } + } + + if (arg.type == "boolean_true") { + arg.default = false + } + if (arg.type == "boolean_false") { + arg.default = true + } + + arg +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/config/addGlobalParams.nf' +def addGlobalArguments(config) { + def localConfig = [ + "argument_groups": [ + [ + "name": "Nextflow input-output arguments", + "description": "Input/output parameters for Nextflow itself. Please note that both publishDir and publish_dir are supported but at least one has to be configured.", + "arguments" : [ + [ + 'name': '--publish_dir', + 'required': true, + 'type': 'string', + 'description': 'Path to an output directory.', + 'example': 'output/', + 'multiple': false + ], + [ + 'name': '--param_list', + 'required': false, + 'type': 'string', + 'description': '''Allows inputting multiple parameter sets to initialise a Nextflow channel. A `param_list` can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. + | + |* A list of maps (as-is) where the keys of each map corresponds to the arguments of the pipeline. Example: in a `nextflow.config` file: `param_list: [ ['id': 'foo', 'input': 'foo.txt'], ['id': 'bar', 'input': 'bar.txt'] ]`. + |* A csv file should have column names which correspond to the different arguments of this pipeline. Example: `--param_list data.csv` with columns `id,input`. + |* A json or a yaml file should be a list of maps, each of which has keys corresponding to the arguments of the pipeline. Example: `--param_list data.json` with contents `[ {'id': 'foo', 'input': 'foo.txt'}, {'id': 'bar', 'input': 'bar.txt'} ]`. + |* A yaml blob can also be passed directly as a string. Example: `--param_list "[ {'id': 'foo', 'input': 'foo.txt'}, {'id': 'bar', 'input': 'bar.txt'} ]"`. + | + |When passing a csv, json or yaml file, relative path names are relativized to the location of the parameter file. No relativation is performed when `param_list` is a list of maps (as-is) or a yaml blob.'''.stripMargin(), + 'example': 'my_params.yaml', + 'multiple': false, + 'hidden': true + ] + // TODO: allow multiple: true in param_list? + // TODO: allow to specify a --param_list_regex to filter the param_list? + // TODO: allow to specify a --param_list_from_state to remap entries in the param_list? + ] + ] + ] + ] + + return processConfig(_mergeMap(config, localConfig)) +} + +def _mergeMap(Map lhs, Map rhs) { + return rhs.inject(lhs.clone()) { map, entry -> + if (map[entry.key] instanceof Map && entry.value instanceof Map) { + map[entry.key] = _mergeMap(map[entry.key], entry.value) + } else if (map[entry.key] instanceof Collection && entry.value instanceof Collection) { + map[entry.key] += entry.value + } else { + map[entry.key] = entry.value + } + return map + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/config/generateHelp.nf' +def _generateArgumentHelp(param) { + // alternatives are not supported + // def names = param.alternatives ::: List(param.name) + + def unnamedProps = [ + ["required parameter", param.required], + ["multiple values allowed", param.multiple], + ["output", param.direction.toLowerCase() == "output"], + ["file must exist", param.type == "file" && param.must_exist] + ].findAll{it[1]}.collect{it[0]} + + def dflt = null + if (param.default != null) { + if (param.default instanceof List) { + dflt = param.default.join(param.multiple_sep != null ? param.multiple_sep : ", ") + } else { + dflt = param.default.toString() + } + } + def example = null + if (param.example != null) { + if (param.example instanceof List) { + example = param.example.join(param.multiple_sep != null ? param.multiple_sep : ", ") + } else { + example = param.example.toString() + } + } + def min = param.min?.toString() + def max = param.max?.toString() + + def escapeChoice = { choice -> + def s1 = choice.replaceAll("\\n", "\\\\n") + def s2 = s1.replaceAll("\"", """\\\"""") + s2.contains(",") || s2 != choice ? "\"" + s2 + "\"" : s2 + } + def choices = param.choices == null ? + null : + "[ " + param.choices.collect{escapeChoice(it.toString())}.join(", ") + " ]" + + def namedPropsStr = [ + ["type", ([param.type] + unnamedProps).join(", ")], + ["default", dflt], + ["example", example], + ["choices", choices], + ["min", min], + ["max", max] + ] + .findAll{it[1]} + .collect{"\n " + it[0] + ": " + it[1].replaceAll("\n", "\\n")} + .join("") + + def descStr = param.description == null ? + "" : + _paragraphWrap("\n" + param.description.trim(), 80 - 8).join("\n ") + + "\n --" + param.plainName + + namedPropsStr + + descStr +} + +// Based on Helper.generateHelp() in Helper.scala +def _generateHelp(config) { + def fun = config + + // PART 1: NAME AND VERSION + def nameStr = fun.name + + (fun.version == null ? "" : " " + fun.version) + + // PART 2: DESCRIPTION + def descrStr = fun.description == null ? + "" : + "\n\n" + _paragraphWrap(fun.description.trim(), 80).join("\n") + + // PART 3: Usage + def usageStr = fun.usage == null ? + "" : + "\n\nUsage:\n" + fun.usage.trim() + + // PART 4: Options + def argGroupStrs = fun.allArgumentGroups.collect{argGroup -> + def name = argGroup.name + def descriptionStr = argGroup.description == null ? + "" : + "\n " + _paragraphWrap(argGroup.description.trim(), 80-4).join("\n ") + "\n" + def arguments = argGroup.arguments.collect{arg -> + arg instanceof String ? fun.allArguments.find{it.plainName == arg} : arg + }.findAll{it != null} + def argumentStrs = arguments.collect{param -> _generateArgumentHelp(param)} + + "\n\n$name:" + + descriptionStr + + argumentStrs.join("\n") + } + + // FINAL: combine + def out = nameStr + + descrStr + + usageStr + + argGroupStrs.join("") + + return out +} + +// based on Format._paragraphWrap +def _paragraphWrap(str, maxLength) { + def outLines = [] + str.split("\n").each{par -> + def words = par.split("\\s").toList() + + def word = null + def line = words.pop() + while(!words.isEmpty()) { + word = words.pop() + if (line.length() + word.length() + 1 <= maxLength) { + line = line + " " + word + } else { + outLines.add(line) + line = word + } + } + if (words.isEmpty()) { + outLines.add(line) + } + } + return outLines +} + +def helpMessage(config) { + if (params.containsKey("help") && params.help) { + def mergedConfig = addGlobalArguments(config) + def helpStr = _generateHelp(mergedConfig) + println(helpStr) + exit 0 + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/config/processConfig.nf' +def processConfig(config) { + // set defaults for arguments + config.arguments = + (config.arguments ?: []).collect{_processArgument(it)} + + // set defaults for argument_group arguments + config.argument_groups = + (config.argument_groups ?: []).collect{grp -> + grp.arguments = (grp.arguments ?: []).collect{_processArgument(it)} + grp + } + + // create combined arguments list + config.allArguments = + config.arguments + + config.argument_groups.collectMany{it.arguments} + + // add missing argument groups (based on Functionality::allArgumentGroups()) + def argGroups = config.argument_groups + if (argGroups.any{it.name.toLowerCase() == "arguments"}) { + argGroups = argGroups.collect{ grp -> + if (grp.name.toLowerCase() == "arguments") { + grp = grp + [ + arguments: grp.arguments + config.arguments + ] + } + grp + } + } else { + argGroups = argGroups + [ + name: "Arguments", + arguments: config.arguments + ] + } + config.allArgumentGroups = argGroups + + config +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/config/readConfig.nf' + +def readConfig(file) { + def config = readYaml(file ?: moduleDir.resolve("config.vsh.yaml")) + processConfig(config) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/_resolveSiblingIfNotAbsolute.nf' +/** + * Resolve a path relative to the current file. + * + * @param str The path to resolve, as a String. + * @param parentPath The path to resolve relative to, as a Path. + * + * @return The path that may have been resovled, as a Path. + */ +def _resolveSiblingIfNotAbsolute(str, parentPath) { + if (str !instanceof String) { + return str + } + if (!_stringIsAbsolutePath(str)) { + return parentPath.resolveSibling(str) + } else { + return file(str, hidden: true) + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/_stringIsAbsolutePath.nf' +/** + * Check whether a path as a string is absolute. + * + * In the past, we tried using `file(., relative: true).isAbsolute()`, + * but the 'relative' option was added in 22.10.0. + * + * @param path The path to check, as a String. + * + * @return Whether the path is absolute, as a boolean. + */ +def _stringIsAbsolutePath(path) { + def _resolve_URL_PROTOCOL = ~/^([a-zA-Z][a-zA-Z0-9]*:)?\\/.+/ + + assert path instanceof String + return _resolve_URL_PROTOCOL.matcher(path).matches() +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/collectTraces.nf' +class CustomTraceObserver implements nextflow.trace.TraceObserver { + List traces + + CustomTraceObserver(List traces) { + this.traces = traces + } + + @Override + void onProcessComplete(nextflow.processor.TaskHandler handler, nextflow.trace.TraceRecord trace) { + def trace2 = trace.store.clone() + trace2.script = null + traces.add(trace2) + } + + @Override + void onProcessCached(nextflow.processor.TaskHandler handler, nextflow.trace.TraceRecord trace) { + def trace2 = trace.store.clone() + trace2.script = null + traces.add(trace2) + } +} + +def collectTraces() { + def traces = Collections.synchronizedList([]) + + // add custom trace observer which stores traces in the traces object + session.observers.add(new CustomTraceObserver(traces)) + + traces +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/deepClone.nf' +/** + * Performs a deep clone of the given object. + * @param x an object + */ +def deepClone(x) { + iterateMap(x, {it instanceof Cloneable ? it.clone() : it}) +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/getPublishDir.nf' +def getPublishDir() { + return params.containsKey("publish_dir") ? params.publish_dir : + params.containsKey("publishDir") ? params.publishDir : + null +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/getRootDir.nf' + +// Recurse upwards until we find a '.build.yaml' file +def _findBuildYamlFile(pathPossiblySymlink) { + def path = pathPossiblySymlink.toRealPath() + def child = path.resolve(".build.yaml") + if (java.nio.file.Files.isDirectory(path) && java.nio.file.Files.exists(child)) { + return child + } else { + def parent = path.getParent() + if (parent == null) { + return null + } else { + return _findBuildYamlFile(parent) + } + } +} + +// get the root of the target folder +def getRootDir() { + def dir = _findBuildYamlFile(meta.resources_dir) + assert dir != null: "Could not find .build.yaml in the folder structure" + dir.getParent() +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/iterateMap.nf' +/** + * Recursively apply a function over the leaves of an object. + * @param obj The object to iterate over. + * @param fun The function to apply to each value. + * @return The object with the function applied to each value. + */ +def iterateMap(obj, fun) { + if (obj instanceof List && obj !instanceof String) { + return obj.collect{item -> + iterateMap(item, fun) + } + } else if (obj instanceof Map) { + return obj.collectEntries{key, item -> + [key.toString(), iterateMap(item, fun)] + } + } else { + return fun(obj) + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/niceView.nf' +/** + * A view for printing the event of each channel as a YAML blob. + * This is useful for debugging. + */ +def niceView() { + workflow niceViewWf { + take: input + main: + output = input + | view{toYamlBlob(it)} + emit: output + } + return niceViewWf +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readCsv.nf' + +def readCsv(file_path) { + def output = [] + def inputFile = file_path !instanceof Path ? file(file_path, hidden: true) : file_path + + // todo: allow escaped quotes in string + // todo: allow single quotes? + def splitRegex = java.util.regex.Pattern.compile(''',(?=(?:[^"]*"[^"]*")*[^"]*$)''') + def removeQuote = java.util.regex.Pattern.compile('''"(.*)"''') + + def br = java.nio.file.Files.newBufferedReader(inputFile) + + def row = -1 + def header = null + while (br.ready() && header == null) { + def line = br.readLine() + row++ + if (!line.startsWith("#")) { + header = splitRegex.split(line, -1).collect{field -> + m = removeQuote.matcher(field) + m.find() ? m.replaceFirst('$1') : field + } + } + } + assert header != null: "CSV file should contain a header" + + while (br.ready()) { + def line = br.readLine() + row++ + if (line == null) { + br.close() + break + } + + if (!line.startsWith("#")) { + def predata = splitRegex.split(line, -1) + def data = predata.collect{field -> + if (field == "") { + return null + } + def m = removeQuote.matcher(field) + if (m.find()) { + return m.replaceFirst('$1') + } else { + return field + } + } + assert header.size() == data.size(): "Row $row should contain the same number as fields as the header" + + def dataMap = [header, data].transpose().collectEntries().findAll{it.value != null} + output.add(dataMap) + } + } + + output +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readJson.nf' +def readJson(file_path) { + def inputFile = file_path !instanceof Path ? file(file_path, hidden: true) : file_path + def jsonSlurper = new groovy.json.JsonSlurper() + jsonSlurper.parse(inputFile) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readJsonBlob.nf' +def readJsonBlob(str) { + def jsonSlurper = new groovy.json.JsonSlurper() + jsonSlurper.parseText(str) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readTaggedYaml.nf' +// Custom constructor to modify how certain objects are parsed from YAML +class CustomConstructor extends org.yaml.snakeyaml.constructor.Constructor { + Path root + + class ConstructPath extends org.yaml.snakeyaml.constructor.AbstractConstruct { + public Object construct(org.yaml.snakeyaml.nodes.Node node) { + String filename = (String) constructScalar(node); + if (root != null) { + return root.resolve(filename); + } + return java.nio.file.Paths.get(filename); + } + } + + CustomConstructor(org.yaml.snakeyaml.LoaderOptions options, Path root) { + super(options) + this.root = root + // Handling !file tag and parse it back to a File type + this.yamlConstructors.put(new org.yaml.snakeyaml.nodes.Tag("!file"), new ConstructPath()) + } +} + +def readTaggedYaml(Path path) { + def options = new org.yaml.snakeyaml.LoaderOptions() + def constructor = new CustomConstructor(options, path.getParent()) + def yaml = new org.yaml.snakeyaml.Yaml(constructor) + return yaml.load(path.text) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readYaml.nf' +def readYaml(file_path) { + def inputFile = file_path !instanceof Path ? file(file_path, hidden: true) : file_path + def yamlSlurper = new org.yaml.snakeyaml.Yaml() + yamlSlurper.load(inputFile) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readYamlBlob.nf' +def readYamlBlob(str) { + def yamlSlurper = new org.yaml.snakeyaml.Yaml() + yamlSlurper.load(str) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/toJsonBlob.nf' +String toJsonBlob(data) { + return groovy.json.JsonOutput.toJson(data) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/toTaggedYamlBlob.nf' +// Custom representer to modify how certain objects are represented in YAML +class CustomRepresenter extends org.yaml.snakeyaml.representer.Representer { + Path relativizer + + class RepresentPath implements org.yaml.snakeyaml.representer.Represent { + public String getFileName(Object obj) { + if (obj instanceof File) { + obj = ((File) obj).toPath(); + } + if (obj !instanceof Path) { + throw new IllegalArgumentException("Object: " + obj + " is not a Path or File"); + } + def path = (Path) obj; + + if (relativizer != null) { + return relativizer.relativize(path).toString() + } else { + return path.toString() + } + } + + public org.yaml.snakeyaml.nodes.Node representData(Object data) { + String filename = getFileName(data); + def tag = new org.yaml.snakeyaml.nodes.Tag("!file"); + return representScalar(tag, filename); + } + } + CustomRepresenter(org.yaml.snakeyaml.DumperOptions options, Path relativizer) { + super(options) + this.relativizer = relativizer + this.representers.put(sun.nio.fs.UnixPath, new RepresentPath()) + this.representers.put(Path, new RepresentPath()) + this.representers.put(File, new RepresentPath()) + } +} + +String toTaggedYamlBlob(data) { + return toRelativeTaggedYamlBlob(data, null) +} +String toRelativeTaggedYamlBlob(data, Path relativizer) { + def options = new org.yaml.snakeyaml.DumperOptions() + options.setDefaultFlowStyle(org.yaml.snakeyaml.DumperOptions.FlowStyle.BLOCK) + def representer = new CustomRepresenter(options, relativizer) + def yaml = new org.yaml.snakeyaml.Yaml(representer, options) + return yaml.dump(data) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/toYamlBlob.nf' +String toYamlBlob(data) { + def options = new org.yaml.snakeyaml.DumperOptions() + options.setDefaultFlowStyle(org.yaml.snakeyaml.DumperOptions.FlowStyle.BLOCK) + options.setPrettyFlow(true) + def yaml = new org.yaml.snakeyaml.Yaml(options) + def cleanData = iterateMap(data, { it instanceof Path ? it.toString() : it }) + return yaml.dump(cleanData) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/writeJson.nf' +void writeJson(data, file) { + assert data: "writeJson: data should not be null" + assert file: "writeJson: file should not be null" + file.write(toJsonBlob(data)) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/writeYaml.nf' +void writeYaml(data, file) { + assert data: "writeYaml: data should not be null" + assert file: "writeYaml: file should not be null" + file.write(toYamlBlob(data)) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/findStates.nf' +def findStates(Map params, Map config) { + def auto_config = deepClone(config) + def auto_params = deepClone(params) + + auto_config = auto_config.clone() + // override arguments + auto_config.argument_groups = [] + auto_config.arguments = [ + [ + type: "string", + name: "--id", + description: "A dummy identifier", + required: false + ], + [ + type: "file", + name: "--input_states", + example: "/path/to/input/directory/**/state.yaml", + description: "Path to input directory containing the datasets to be integrated.", + required: true, + multiple: true, + multiple_sep: ";" + ], + [ + type: "string", + name: "--filter", + example: "foo/.*/state.yaml", + description: "Regex to filter state files by path.", + required: false + ], + // to do: make this a yaml blob? + [ + type: "string", + name: "--rename_keys", + example: ["newKey1:oldKey1", "newKey2:oldKey2"], + description: "Rename keys in the detected input files. This is useful if the input files do not match the set of input arguments of the workflow.", + required: false, + multiple: true, + multiple_sep: ";" + ], + [ + type: "string", + name: "--settings", + example: '{"output_dataset": "dataset.h5ad", "k": 10}', + description: "Global arguments as a JSON glob to be passed to all components.", + required: false + ] + ] + if (!(auto_params.containsKey("id"))) { + auto_params["id"] = "auto" + } + + // run auto config through processConfig once more + auto_config = processConfig(auto_config) + + workflow findStatesWf { + helpMessage(auto_config) + + output_ch = + channelFromParams(auto_params, auto_config) + | flatMap { autoId, args -> + + def globalSettings = args.settings ? readYamlBlob(args.settings) : [:] + + // look for state files in input dir + def stateFiles = args.input_states + + // filter state files by regex + if (args.filter) { + stateFiles = stateFiles.findAll{ stateFile -> + def stateFileStr = stateFile.toString() + def matcher = stateFileStr =~ args.filter + matcher.matches()} + } + + // read in states + def states = stateFiles.collect { stateFile -> + def state_ = readTaggedYaml(stateFile) + [state_.id, state_] + } + + // construct renameMap + if (args.rename_keys) { + def renameMap = args.rename_keys.collectEntries{renameString -> + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" + split + } + + // rename keys in state, only let states through which have all keys + // also add global settings + states = states.collectMany{id, state -> + def newState = [:] + + for (key in renameMap.keySet()) { + def origKey = renameMap[key] + if (!(state.containsKey(origKey))) { + return [] + } + newState[key] = state[origKey] + } + + [[id, globalSettings + newState]] + } + } + + states + } + emit: + output_ch + } + + return findStatesWf +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/joinStates.nf' +def joinStates(Closure apply_) { + workflow joinStatesWf { + take: input_ch + main: + output_ch = input_ch + | toSortedList + | filter{ it.size() > 0 } + | map{ tups -> + def ids = tups.collect{it[0]} + def states = tups.collect{it[1]} + apply_(ids, states) + } + + emit: output_ch + } + return joinStatesWf +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishStates.nf' +def collectFiles(obj) { + if (obj instanceof java.io.File || obj instanceof Path) { + return [obj] + } else if (obj instanceof List && obj !instanceof String) { + return obj.collectMany{item -> + collectFiles(item) + } + } else if (obj instanceof Map) { + return obj.collectMany{key, item -> + collectFiles(item) + } + } else { + return [] + } +} + +/** + * Recurse through a state and collect all input files and their target output filenames. + * @param obj The state to recurse through. + * @param prefix The prefix to prepend to the output filenames. + */ +def collectInputOutputPaths(obj, prefix) { + if (obj instanceof File || obj instanceof Path) { + def path = obj instanceof Path ? obj : obj.toPath() + def ext = path.getFileName().toString().find("\\.[^\\.]+\$") ?: "" + def newFilename = prefix + ext + return [[obj, newFilename]] + } else if (obj instanceof List && obj !instanceof String) { + return obj.withIndex().collectMany{item, ix -> + collectInputOutputPaths(item, prefix + "_" + ix) + } + } else if (obj instanceof Map) { + return obj.collectMany{key, item -> + collectInputOutputPaths(item, prefix + "." + key) + } + } else { + return [] + } +} + +def publishStates(Map args) { + def key_ = args.get("key") + def yamlTemplate_ = args.get("output_state", args.get("outputState", '$id.$key.state.yaml')) + + assert key_ != null : "publishStates: key must be specified" + + workflow publishStatesWf { + take: input_ch + main: + input_ch + | map { tup -> + def id_ = tup[0] + def state_ = tup[1] + + // the input files and the target output filenames + def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose() + def inputFiles_ = inputoutputFilenames_[0] + def outputFilenames_ = inputoutputFilenames_[1] + + def yamlFilename = yamlTemplate_ + .replaceAll('\\$id', id_) + .replaceAll('\\$key', key_) + + // TODO: do the pathnames in state_ match up with the outputFilenames_? + + // convert state to yaml blob + def yamlBlob_ = toRelativeTaggedYamlBlob([id: id_] + state_, java.nio.file.Paths.get(yamlFilename)) + + [id_, yamlBlob_, yamlFilename, inputFiles_, outputFilenames_] + } + | publishStatesProc + emit: input_ch + } + return publishStatesWf +} +process publishStatesProc { + // todo: check publishpath? + publishDir path: "${getPublishDir()}/", mode: "copy" + tag "$id" + input: + tuple val(id), val(yamlBlob), val(yamlFile), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles) + output: + tuple val(id), path{[yamlFile] + outputFiles} + script: + def copyCommands = [ + inputFiles instanceof List ? inputFiles : [inputFiles], + outputFiles instanceof List ? outputFiles : [outputFiles] + ] + .transpose() + .collectMany{infile, outfile -> + if (infile.toString() != outfile.toString()) { + [ + "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"", + "cp -r '${infile.toString()}' '${outfile.toString()}'" + ] + } else { + // no need to copy if infile is the same as outfile + [] + } + } + """ +mkdir -p "\$(dirname '${yamlFile}')" +echo "Storing state as yaml" +echo '${yamlBlob}' > '${yamlFile}' +echo "Copying output files to destination folder" +${copyCommands.join("\n ")} +""" +} + + +// this assumes that the state contains no other values other than those specified in the config +def publishStatesByConfig(Map args) { + def config = args.get("config") + assert config != null : "publishStatesByConfig: config must be specified" + + def key_ = args.get("key", config.name) + assert key_ != null : "publishStatesByConfig: key must be specified" + + workflow publishStatesSimpleWf { + take: input_ch + main: + input_ch + | map { tup -> + def id_ = tup[0] + def state_ = tup[1] // e.g. [output: new File("myoutput.h5ad"), k: 10] + def origState_ = tup[2] // e.g. [output: '$id.$key.foo.h5ad'] + + // TODO: allow overriding the state.yaml template + // TODO TODO: if auto.publish == "state", add output_state as an argument + def yamlTemplate = params.containsKey("output_state") ? params.output_state : '$id.$key.state.yaml' + def yamlFilename = yamlTemplate + .replaceAll('\\$id', id_) + .replaceAll('\\$key', key_) + def yamlDir = java.nio.file.Paths.get(yamlFilename).getParent() + + // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where + // - key is a String + // - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path) + // - inputPath is a List[Path] + // - outputFilename is a List[String] + // - (key, value) are the tuples that will be saved to the state.yaml file + // - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml) + def processedState = + config.allArguments + .findAll { it.direction == "output" } + .collectMany { par -> + def plainName_ = par.plainName + // if the state does not contain the key, it's an + // optional argument for which the component did + // not generate any output + if (!state_.containsKey(plainName_)) { + return [] + } + def value = state_[plainName_] + // if the parameter is not a file, it should be stored + // in the state as-is, but is not something that needs + // to be copied from the source path to the dest path + if (par.type != "file") { + return [[key: plainName_, value: value, inputPath: [], outputFilename: []]] + } + // if the orig state does not contain this filename, + // it's an optional argument for which the user specified + // that it should not be returned as a state + if (!origState_.containsKey(plainName_)) { + return [] + } + def filenameTemplate = origState_[plainName_] + // if the pararameter is multiple: true, fetch the template + if (par.multiple && filenameTemplate instanceof List) { + filenameTemplate = filenameTemplate[0] + } + // instantiate the template + def filename = filenameTemplate + .replaceAll('\\$id', id_) + .replaceAll('\\$key', key_) + if (par.multiple) { + // if the parameter is multiple: true, the filename + // should contain a wildcard '*' that is replaced with + // the index of the file + assert filename.contains("*") : "Module '${key_}' id '${id_}': Multiple output files specified, but no wildcard '*' in the filename: ${filename}" + def outputPerFile = value.withIndex().collect{ val, ix -> + def filename_ix = filename.replace("*", ix.toString()) + def value_ = java.nio.file.Paths.get(filename_ix) + // if id contains a slash + if (yamlDir != null) { + value_ = yamlDir.relativize(value_) + } + def inputPath = val instanceof File ? val.toPath() : val + [value: value_, inputPath: inputPath, outputFilename: filename_ix] + } + def transposedOutputs = ["value", "inputPath", "outputFilename"].collectEntries{ key -> + [key, outputPerFile.collect{dic -> dic[key]}] + } + return [[key: plainName_] + transposedOutputs] + } else { + def value_ = java.nio.file.Paths.get(filename) + // if id contains a slash + if (yamlDir != null) { + value_ = yamlDir.relativize(value_) + } + def inputPath = value instanceof File ? value.toPath() : value + return [[key: plainName_, value: value_, inputPath: [inputPath], outputFilename: [filename]]] + } + } + + def updatedState_ = processedState.collectEntries{[it.key, it.value]} + def inputPaths = processedState.collectMany{it.inputPath} + def outputFilenames = processedState.collectMany{it.outputFilename} + + // convert state to yaml blob + def yamlBlob_ = toTaggedYamlBlob([id: id_] + updatedState_) + + [id_, yamlBlob_, yamlFilename, inputPaths, outputFilenames] + } + | publishStatesProc + emit: input_ch + } + return publishStatesSimpleWf +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/setState.nf' +def setState(fun) { + assert fun instanceof Closure || fun instanceof Map || fun instanceof List : + "Error in setState: Expected process argument to be a Closure, a Map, or a List. Found: class ${fun.getClass()}" + + // if fun is a List, convert to map + if (fun instanceof List) { + // check whether fun is a list[string] + assert fun.every{it instanceof CharSequence} : "Error in setState: argument is a List, but not all elements are Strings" + fun = fun.collectEntries{[it, it]} + } + + // if fun is a map, convert to closure + if (fun instanceof Map) { + // check whether fun is a map[string, string] + assert fun.values().every{it instanceof CharSequence} : "Error in setState: argument is a Map, but not all values are Strings" + assert fun.keySet().every{it instanceof CharSequence} : "Error in setState: argument is a Map, but not all keys are Strings" + def funMap = fun.clone() + // turn the map into a closure to be used later on + fun = { id_, state_ -> + assert state_ instanceof Map : "Error in setState: the state is not a Map" + funMap.collectMany{newkey, origkey -> + if (state_.containsKey(origkey)) { + [[newkey, state_[origkey]]] + } else { + [] + } + }.collectEntries() + } + } + + map { tup -> + def id = tup[0] + def state = tup[1] + def unfilteredState = fun(id, state) + def newState = unfilteredState.findAll{key, val -> val != null} + [id, newState] + tup.drop(2) + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/processAuto.nf' +// TODO: unit test processAuto +def processAuto(Map auto) { + // remove null values + auto = auto.findAll{k, v -> v != null} + + // check for unexpected keys + def expectedKeys = ["simplifyInput", "simplifyOutput", "transcript", "publish"] + def unexpectedKeys = auto.keySet() - expectedKeys + assert unexpectedKeys.isEmpty(), "unexpected keys in auto: '${unexpectedKeys.join("', '")}'" + + // check auto.simplifyInput + assert auto.simplifyInput instanceof Boolean, "auto.simplifyInput must be a boolean" + + // check auto.simplifyOutput + assert auto.simplifyOutput instanceof Boolean, "auto.simplifyOutput must be a boolean" + + // check auto.transcript + assert auto.transcript instanceof Boolean, "auto.transcript must be a boolean" + + // check auto.publish + assert auto.publish instanceof Boolean || auto.publish == "state", "auto.publish must be a boolean or 'state'" + + return auto.subMap(expectedKeys) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/processDirectives.nf' +def assertMapKeys(map, expectedKeys, requiredKeys, mapName) { + assert map instanceof Map : "Expected argument '$mapName' to be a Map. Found: class ${map.getClass()}" + map.forEach { key, val -> + assert key in expectedKeys : "Unexpected key '$key' in ${mapName ? mapName + " " : ""}map" + } + requiredKeys.forEach { requiredKey -> + assert map.containsKey(requiredKey) : "Missing required key '$key' in ${mapName ? mapName + " " : ""}map" + } +} + +// TODO: unit test processDirectives +def processDirectives(Map drctv) { + // remove null values + drctv = drctv.findAll{k, v -> v != null} + + // check for unexpected keys + def expectedKeys = [ + "accelerator", "afterScript", "beforeScript", "cache", "conda", "container", "containerOptions", "cpus", "disk", "echo", "errorStrategy", "executor", "machineType", "maxErrors", "maxForks", "maxRetries", "memory", "module", "penv", "pod", "publishDir", "queue", "label", "scratch", "storeDir", "stageInMode", "stageOutMode", "tag", "time" + ] + def unexpectedKeys = drctv.keySet() - expectedKeys + assert unexpectedKeys.isEmpty() : "Unexpected keys in process directive: '${unexpectedKeys.join("', '")}'" + + /* DIRECTIVE accelerator + accepted examples: + - [ limit: 4, type: "nvidia-tesla-k80" ] + */ + if (drctv.containsKey("accelerator")) { + assertMapKeys(drctv["accelerator"], ["type", "limit", "request", "runtime"], [], "accelerator") + } + + /* DIRECTIVE afterScript + accepted examples: + - "source /cluster/bin/cleanup" + */ + if (drctv.containsKey("afterScript")) { + assert drctv["afterScript"] instanceof CharSequence + } + + /* DIRECTIVE beforeScript + accepted examples: + - "source /cluster/bin/setup" + */ + if (drctv.containsKey("beforeScript")) { + assert drctv["beforeScript"] instanceof CharSequence + } + + /* DIRECTIVE cache + accepted examples: + - true + - false + - "deep" + - "lenient" + */ + if (drctv.containsKey("cache")) { + assert drctv["cache"] instanceof CharSequence || drctv["cache"] instanceof Boolean + if (drctv["cache"] instanceof CharSequence) { + assert drctv["cache"] in ["deep", "lenient"] : "Unexpected value for cache" + } + } + + /* DIRECTIVE conda + accepted examples: + - "bwa=0.7.15" + - "bwa=0.7.15 fastqc=0.11.5" + - ["bwa=0.7.15", "fastqc=0.11.5"] + */ + if (drctv.containsKey("conda")) { + if (drctv["conda"] instanceof List) { + drctv["conda"] = drctv["conda"].join(" ") + } + assert drctv["conda"] instanceof CharSequence + } + + /* DIRECTIVE container + accepted examples: + - "foo/bar:tag" + - [ registry: "reg", image: "im", tag: "ta" ] + is transformed to "reg/im:ta" + - [ image: "im" ] + is transformed to "im:latest" + */ + if (drctv.containsKey("container")) { + assert drctv["container"] instanceof Map || drctv["container"] instanceof CharSequence + if (drctv["container"] instanceof Map) { + def m = drctv["container"] + assertMapKeys(m, [ "registry", "image", "tag" ], ["image"], "container") + def part1 = + System.getenv('OVERRIDE_CONTAINER_REGISTRY') ? System.getenv('OVERRIDE_CONTAINER_REGISTRY') + "/" : + params.containsKey("override_container_registry") ? params["override_container_registry"] + "/" : // todo: remove? + m.registry ? m.registry + "/" : + "" + def part2 = m.image + def part3 = m.tag ? ":" + m.tag : ":latest" + drctv["container"] = part1 + part2 + part3 + } + } + + /* DIRECTIVE containerOptions + accepted examples: + - "--foo bar" + - ["--foo bar", "-f b"] + */ + if (drctv.containsKey("containerOptions")) { + if (drctv["containerOptions"] instanceof List) { + drctv["containerOptions"] = drctv["containerOptions"].join(" ") + } + assert drctv["containerOptions"] instanceof CharSequence + } + + /* DIRECTIVE cpus + accepted examples: + - 1 + - 10 + */ + if (drctv.containsKey("cpus")) { + assert drctv["cpus"] instanceof Integer + } + + /* DIRECTIVE disk + accepted examples: + - "1 GB" + - "2TB" + - "3.2KB" + - "10.B" + */ + if (drctv.containsKey("disk")) { + assert drctv["disk"] instanceof CharSequence + // assert drctv["disk"].matches("[0-9]+(\\.[0-9]*)? *[KMGTPEZY]?B") + // ^ does not allow closures + } + + /* DIRECTIVE echo + accepted examples: + - true + - false + */ + if (drctv.containsKey("echo")) { + assert drctv["echo"] instanceof Boolean + } + + /* DIRECTIVE errorStrategy + accepted examples: + - "terminate" + - "finish" + */ + if (drctv.containsKey("errorStrategy")) { + assert drctv["errorStrategy"] instanceof CharSequence + assert drctv["errorStrategy"] in ["terminate", "finish", "ignore", "retry"] : "Unexpected value for errorStrategy" + } + + /* DIRECTIVE executor + accepted examples: + - "local" + - "sge" + */ + if (drctv.containsKey("executor")) { + assert drctv["executor"] instanceof CharSequence + assert drctv["executor"] in ["local", "sge", "uge", "lsf", "slurm", "pbs", "pbspro", "moab", "condor", "nqsii", "ignite", "k8s", "awsbatch", "google-pipelines"] : "Unexpected value for executor" + } + + /* DIRECTIVE machineType + accepted examples: + - "n1-highmem-8" + */ + if (drctv.containsKey("machineType")) { + assert drctv["machineType"] instanceof CharSequence + } + + /* DIRECTIVE maxErrors + accepted examples: + - 1 + - 3 + */ + if (drctv.containsKey("maxErrors")) { + assert drctv["maxErrors"] instanceof Integer + } + + /* DIRECTIVE maxForks + accepted examples: + - 1 + - 3 + */ + if (drctv.containsKey("maxForks")) { + assert drctv["maxForks"] instanceof Integer + } + + /* DIRECTIVE maxRetries + accepted examples: + - 1 + - 3 + */ + if (drctv.containsKey("maxRetries")) { + assert drctv["maxRetries"] instanceof Integer + } + + /* DIRECTIVE memory + accepted examples: + - "1 GB" + - "2TB" + - "3.2KB" + - "10.B" + */ + if (drctv.containsKey("memory")) { + assert drctv["memory"] instanceof CharSequence + // assert drctv["memory"].matches("[0-9]+(\\.[0-9]*)? *[KMGTPEZY]?B") + // ^ does not allow closures + } + + /* DIRECTIVE module + accepted examples: + - "ncbi-blast/2.2.27" + - "ncbi-blast/2.2.27:t_coffee/10.0" + - ["ncbi-blast/2.2.27", "t_coffee/10.0"] + */ + if (drctv.containsKey("module")) { + if (drctv["module"] instanceof List) { + drctv["module"] = drctv["module"].join(":") + } + assert drctv["module"] instanceof CharSequence + } + + /* DIRECTIVE penv + accepted examples: + - "smp" + */ + if (drctv.containsKey("penv")) { + assert drctv["penv"] instanceof CharSequence + } + + /* DIRECTIVE pod + accepted examples: + - [ label: "key", value: "val" ] + - [ annotation: "key", value: "val" ] + - [ env: "key", value: "val" ] + - [ [label: "l", value: "v"], [env: "e", value: "v"]] + */ + if (drctv.containsKey("pod")) { + if (drctv["pod"] instanceof Map) { + drctv["pod"] = [ drctv["pod"] ] + } + assert drctv["pod"] instanceof List + drctv["pod"].forEach { pod -> + assert pod instanceof Map + // TODO: should more checks be added? + // See https://www.nextflow.io/docs/latest/process.html?highlight=directives#pod + // e.g. does it contain 'label' and 'value', or 'annotation' and 'value', or ...? + } + } + + /* DIRECTIVE publishDir + accepted examples: + - [] + - [ [ path: "foo", enabled: true ], [ path: "bar", enabled: false ] ] + - "/path/to/dir" + is transformed to [[ path: "/path/to/dir" ]] + - [ path: "/path/to/dir", mode: "cache" ] + is transformed to [[ path: "/path/to/dir", mode: "cache" ]] + */ + // TODO: should we also look at params["publishDir"]? + if (drctv.containsKey("publishDir")) { + def pblsh = drctv["publishDir"] + + // check different options + assert pblsh instanceof List || pblsh instanceof Map || pblsh instanceof CharSequence + + // turn into list if not already so + // for some reason, 'if (!pblsh instanceof List) pblsh = [ pblsh ]' doesn't work. + pblsh = pblsh instanceof List ? pblsh : [ pblsh ] + + // check elements of publishDir + pblsh = pblsh.collect{ elem -> + // turn into map if not already so + elem = elem instanceof CharSequence ? [ path: elem ] : elem + + // check types and keys + assert elem instanceof Map : "Expected publish argument '$elem' to be a String or a Map. Found: class ${elem.getClass()}" + assertMapKeys(elem, [ "path", "mode", "overwrite", "pattern", "saveAs", "enabled" ], ["path"], "publishDir") + + // check elements in map + assert elem.containsKey("path") + assert elem["path"] instanceof CharSequence + if (elem.containsKey("mode")) { + assert elem["mode"] instanceof CharSequence + assert elem["mode"] in [ "symlink", "rellink", "link", "copy", "copyNoFollow", "move" ] + } + if (elem.containsKey("overwrite")) { + assert elem["overwrite"] instanceof Boolean + } + if (elem.containsKey("pattern")) { + assert elem["pattern"] instanceof CharSequence + } + if (elem.containsKey("saveAs")) { + assert elem["saveAs"] instanceof CharSequence //: "saveAs as a Closure is currently not supported. Surround your closure with single quotes to get the desired effect. Example: '\{ foo \}'" + } + if (elem.containsKey("enabled")) { + assert elem["enabled"] instanceof Boolean + } + + // return final result + elem + } + // store final directive + drctv["publishDir"] = pblsh + } + + /* DIRECTIVE queue + accepted examples: + - "long" + - "short,long" + - ["short", "long"] + */ + if (drctv.containsKey("queue")) { + if (drctv["queue"] instanceof List) { + drctv["queue"] = drctv["queue"].join(",") + } + assert drctv["queue"] instanceof CharSequence + } + + /* DIRECTIVE label + accepted examples: + - "big_mem" + - "big_cpu" + - ["big_mem", "big_cpu"] + */ + if (drctv.containsKey("label")) { + if (drctv["label"] instanceof CharSequence) { + drctv["label"] = [ drctv["label"] ] + } + assert drctv["label"] instanceof List + drctv["label"].forEach { label -> + assert label instanceof CharSequence + // assert label.matches("[a-zA-Z0-9]([a-zA-Z0-9_]*[a-zA-Z0-9])?") + // ^ does not allow closures + } + } + + /* DIRECTIVE scratch + accepted examples: + - true + - "/path/to/scratch" + - '$MY_PATH_TO_SCRATCH' + - "ram-disk" + */ + if (drctv.containsKey("scratch")) { + assert drctv["scratch"] == true || drctv["scratch"] instanceof CharSequence + } + + /* DIRECTIVE storeDir + accepted examples: + - "/path/to/storeDir" + */ + if (drctv.containsKey("storeDir")) { + assert drctv["storeDir"] instanceof CharSequence + } + + /* DIRECTIVE stageInMode + accepted examples: + - "copy" + - "link" + */ + if (drctv.containsKey("stageInMode")) { + assert drctv["stageInMode"] instanceof CharSequence + assert drctv["stageInMode"] in ["copy", "link", "symlink", "rellink"] + } + + /* DIRECTIVE stageOutMode + accepted examples: + - "copy" + - "link" + */ + if (drctv.containsKey("stageOutMode")) { + assert drctv["stageOutMode"] instanceof CharSequence + assert drctv["stageOutMode"] in ["copy", "move", "rsync"] + } + + /* DIRECTIVE tag + accepted examples: + - "foo" + - '$id' + */ + if (drctv.containsKey("tag")) { + assert drctv["tag"] instanceof CharSequence + } + + /* DIRECTIVE time + accepted examples: + - "1h" + - "2days" + - "1day 6hours 3minutes 30seconds" + */ + if (drctv.containsKey("time")) { + assert drctv["time"] instanceof CharSequence + // todo: validation regex? + } + + return drctv +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/processWorkflowArgs.nf' +def processWorkflowArgs(Map args, Map defaultWfArgs, Map meta) { + // override defaults with args + def workflowArgs = defaultWfArgs + args + + // check whether 'key' exists + assert workflowArgs.containsKey("key") : "Error in module '${meta.config.name}': key is a required argument" + + // if 'key' is a closure, apply it to the original key + if (workflowArgs["key"] instanceof Closure) { + workflowArgs["key"] = workflowArgs["key"](meta.config.name) + } + def key = workflowArgs["key"] + assert key instanceof CharSequence : "Expected process argument 'key' to be a String. Found: class ${key.getClass()}" + assert key ==~ /^[a-zA-Z_]\w*$/ : "Error in module '$key': Expected process argument 'key' to consist of only letters, digits or underscores. Found: ${key}" + + // check for any unexpected keys + def expectedKeys = ["key", "directives", "auto", "map", "mapId", "mapData", "mapPassthrough", "filter", "runIf", "fromState", "toState", "args", "renameKeys", "debug"] + def unexpectedKeys = workflowArgs.keySet() - expectedKeys + assert unexpectedKeys.isEmpty() : "Error in module '$key': unexpected arguments to the '.run()' function: '${unexpectedKeys.join("', '")}'" + + // check whether directives exists and apply defaults + assert workflowArgs.containsKey("directives") : "Error in module '$key': directives is a required argument" + assert workflowArgs["directives"] instanceof Map : "Error in module '$key': Expected process argument 'directives' to be a Map. Found: class ${workflowArgs['directives'].getClass()}" + workflowArgs["directives"] = processDirectives(defaultWfArgs.directives + workflowArgs["directives"]) + + // check whether directives exists and apply defaults + assert workflowArgs.containsKey("auto") : "Error in module '$key': auto is a required argument" + assert workflowArgs["auto"] instanceof Map : "Error in module '$key': Expected process argument 'auto' to be a Map. Found: class ${workflowArgs['auto'].getClass()}" + workflowArgs["auto"] = processAuto(defaultWfArgs.auto + workflowArgs["auto"]) + + // auto define publish, if so desired + if (workflowArgs.auto.publish == true && (workflowArgs.directives.publishDir != null ? workflowArgs.directives.publishDir : [:]).isEmpty()) { + // can't assert at this level thanks to the no_publish profile + // assert params.containsKey("publishDir") || params.containsKey("publish_dir") : + // "Error in module '${workflowArgs['key']}': if auto.publish is true, params.publish_dir needs to be defined.\n" + + // " Example: params.publish_dir = \"./output/\"" + def publishDir = getPublishDir() + + if (publishDir != null) { + workflowArgs.directives.publishDir = [[ + path: publishDir, + saveAs: "{ it.startsWith('.') ? null : it }", // don't publish hidden files, by default + mode: "copy" + ]] + } + } + + // auto define transcript, if so desired + if (workflowArgs.auto.transcript == true) { + // can't assert at this level thanks to the no_publish profile + // assert params.containsKey("transcriptsDir") || params.containsKey("transcripts_dir") || params.containsKey("publishDir") || params.containsKey("publish_dir") : + // "Error in module '${workflowArgs['key']}': if auto.transcript is true, either params.transcripts_dir or params.publish_dir needs to be defined.\n" + + // " Example: params.transcripts_dir = \"./transcripts/\"" + def transcriptsDir = + params.containsKey("transcripts_dir") ? params.transcripts_dir : + params.containsKey("transcriptsDir") ? params.transcriptsDir : + params.containsKey("publish_dir") ? params.publish_dir + "/_transcripts" : + params.containsKey("publishDir") ? params.publishDir + "/_transcripts" : + null + if (transcriptsDir != null) { + def timestamp = nextflow.Nextflow.getSession().getWorkflowMetadata().start.format('yyyy-MM-dd_HH-mm-ss') + def transcriptsPublishDir = [ + path: "$transcriptsDir/$timestamp/\${task.process.replaceAll(':', '-')}/\${id}/", + saveAs: "{ it.startsWith('.') ? it.replaceAll('^.', '') : null }", + mode: "copy" + ] + def publishDirs = workflowArgs.directives.publishDir != null ? workflowArgs.directives.publishDir : null ? workflowArgs.directives.publishDir : [] + workflowArgs.directives.publishDir = publishDirs + transcriptsPublishDir + } + } + + // if this is a stubrun, remove certain directives? + if (workflow.stubRun) { + workflowArgs.directives.keySet().removeAll(["publishDir", "cpus", "memory", "label"]) + } + + for (nam in ["map", "mapId", "mapData", "mapPassthrough", "filter", "runIf"]) { + if (workflowArgs.containsKey(nam) && workflowArgs[nam]) { + assert workflowArgs[nam] instanceof Closure : "Error in module '$key': Expected process argument '$nam' to be null or a Closure. Found: class ${workflowArgs[nam].getClass()}" + } + } + + // TODO: should functions like 'map', 'mapId', 'mapData', 'mapPassthrough' be deprecated as well? + for (nam in ["map", "mapData", "mapPassthrough", "renameKeys"]) { + if (workflowArgs.containsKey(nam) && workflowArgs[nam] != null) { + log.warn "module '$key': workflow argument '$nam' is deprecated and will be removed in Viash 0.9.0. Please use 'fromState' and 'toState' instead." + } + } + + // check fromState + workflowArgs["fromState"] = _processFromState(workflowArgs.get("fromState"), key, meta.config) + + // check toState + workflowArgs["toState"] = _processToState(workflowArgs.get("toState"), key, meta.config) + + // return output + return workflowArgs +} + +def _processFromState(fromState, key_, config_) { + assert fromState == null || fromState instanceof Closure || fromState instanceof Map || fromState instanceof List : + "Error in module '$key_': Expected process argument 'fromState' to be null, a Closure, a Map, or a List. Found: class ${fromState.getClass()}" + if (fromState == null) { + return null + } + + // if fromState is a List, convert to map + if (fromState instanceof List) { + // check whether fromstate is a list[string] + assert fromState.every{it instanceof CharSequence} : "Error in module '$key_': fromState is a List, but not all elements are Strings" + fromState = fromState.collectEntries{[it, it]} + } + + // if fromState is a map, convert to closure + if (fromState instanceof Map) { + // check whether fromstate is a map[string, string] + assert fromState.values().every{it instanceof CharSequence} : "Error in module '$key_': fromState is a Map, but not all values are Strings" + assert fromState.keySet().every{it instanceof CharSequence} : "Error in module '$key_': fromState is a Map, but not all keys are Strings" + def fromStateMap = fromState.clone() + def requiredInputNames = meta.config.allArguments.findAll{it.required && it.direction == "Input"}.collect{it.plainName} + // turn the map into a closure to be used later on + fromState = { it -> + def state = it[1] + assert state instanceof Map : "Error in module '$key_': the state is not a Map" + def data = fromStateMap.collectMany{newkey, origkey -> + // check whether newkey corresponds to a required argument + if (state.containsKey(origkey)) { + [[newkey, state[origkey]]] + } else if (!requiredInputNames.contains(origkey)) { + [] + } else { + throw new Exception("Error in module '$key_': fromState key '$origkey' not found in current state") + } + }.collectEntries() + data + } + } + + return fromState +} + +def _processToState(toState, key_, config_) { + if (toState == null) { + toState = { tup -> tup[1] } + } + + // toState should be a closure, map[string, string], or list[string] + assert toState instanceof Closure || toState instanceof Map || toState instanceof List : + "Error in module '$key_': Expected process argument 'toState' to be a Closure, a Map, or a List. Found: class ${toState.getClass()}" + + // if toState is a List, convert to map + if (toState instanceof List) { + // check whether toState is a list[string] + assert toState.every{it instanceof CharSequence} : "Error in module '$key_': toState is a List, but not all elements are Strings" + toState = toState.collectEntries{[it, it]} + } + + // if toState is a map, convert to closure + if (toState instanceof Map) { + // check whether toState is a map[string, string] + assert toState.values().every{it instanceof CharSequence} : "Error in module '$key_': toState is a Map, but not all values are Strings" + assert toState.keySet().every{it instanceof CharSequence} : "Error in module '$key_': toState is a Map, but not all keys are Strings" + def toStateMap = toState.clone() + def requiredOutputNames = config_.allArguments.findAll{it.required && it.direction == "Output"}.collect{it.plainName} + // turn the map into a closure to be used later on + toState = { it -> + def output = it[1] + def state = it[2] + assert output instanceof Map : "Error in module '$key_': the output is not a Map" + assert state instanceof Map : "Error in module '$key_': the state is not a Map" + def extraEntries = toStateMap.collectMany{newkey, origkey -> + // check whether newkey corresponds to a required argument + if (output.containsKey(origkey)) { + [[newkey, output[origkey]]] + } else if (!requiredOutputNames.contains(origkey)) { + [] + } else { + throw new Exception("Error in module '$key_': toState key '$origkey' not found in current output") + } + }.collectEntries() + state + extraEntries + } + } + + return toState +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/workflowFactory.nf' +def _debug(workflowArgs, debugKey) { + if (workflowArgs.debug) { + view { "process '${workflowArgs.key}' $debugKey tuple: $it" } + } else { + map { it } + } +} + +// depends on: innerWorkflowFactory +def workflowFactory(Map args, Map defaultWfArgs, Map meta) { + def workflowArgs = processWorkflowArgs(args, defaultWfArgs, meta) + def key_ = workflowArgs["key"] + + workflow workflowInstance { + take: input_ + + main: + def chModified = input_ + | checkUniqueIds([:]) + | _debug(workflowArgs, "input") + | map { tuple -> + tuple = deepClone(tuple) + + if (workflowArgs.map) { + tuple = workflowArgs.map(tuple) + } + if (workflowArgs.mapId) { + tuple[0] = workflowArgs.mapId(tuple[0]) + } + if (workflowArgs.mapData) { + tuple[1] = workflowArgs.mapData(tuple[1]) + } + if (workflowArgs.mapPassthrough) { + tuple = tuple.take(2) + workflowArgs.mapPassthrough(tuple.drop(2)) + } + + // check tuple + assert tuple instanceof List : + "Error in module '${key_}': element in channel should be a tuple [id, data, ...otherargs...]\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Expected class: List. Found: tuple.getClass() is ${tuple.getClass()}" + assert tuple.size() >= 2 : + "Error in module '${key_}': expected length of tuple in input channel to be two or greater.\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Found: tuple.size() == ${tuple.size()}" + + // check id field + if (tuple[0] instanceof GString) { + tuple[0] = tuple[0].toString() + } + assert tuple[0] instanceof CharSequence : + "Error in module '${key_}': first element of tuple in channel should be a String\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Found: ${tuple[0]}" + + // match file to input file + if (workflowArgs.auto.simplifyInput && (tuple[1] instanceof Path || tuple[1] instanceof List)) { + def inputFiles = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "input" } + + assert inputFiles.size() == 1 : + "Error in module '${key_}' id '${tuple[0]}'.\n" + + " Anonymous file inputs are only allowed when the process has exactly one file input.\n" + + " Expected: inputFiles.size() == 1. Found: inputFiles.size() is ${inputFiles.size()}" + + tuple[1] = [[ inputFiles[0].plainName, tuple[1] ]].collectEntries() + } + + // check data field + assert tuple[1] instanceof Map : + "Error in module '${key_}' id '${tuple[0]}': second element of tuple in channel should be a Map\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Expected class: Map. Found: tuple[1].getClass() is ${tuple[1].getClass()}" + + // rename keys of data field in tuple + if (workflowArgs.renameKeys) { + assert workflowArgs.renameKeys instanceof Map : + "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + + " Example: renameKeys: ['new_key': 'old_key'].\n" + + " Expected class: Map. Found: renameKeys.getClass() is ${workflowArgs.renameKeys.getClass()}" + assert tuple[1] instanceof Map : + "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + + " Expected class: Map. Found: tuple[1].getClass() is ${tuple[1].getClass()}" + + // TODO: allow renameKeys to be a function? + workflowArgs.renameKeys.each { newKey, oldKey -> + assert newKey instanceof CharSequence : + "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + + " Example: renameKeys: ['new_key': 'old_key'].\n" + + " Expected class of newKey: String. Found: newKey.getClass() is ${newKey.getClass()}" + assert oldKey instanceof CharSequence : + "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + + " Example: renameKeys: ['new_key': 'old_key'].\n" + + " Expected class of oldKey: String. Found: oldKey.getClass() is ${oldKey.getClass()}" + assert tuple[1].containsKey(oldKey) : + "Error renaming data keys in module '${key}' id '${tuple[0]}'.\n" + + " Key '$oldKey' is missing in the data map. tuple[1].keySet() is '${tuple[1].keySet()}'" + tuple[1].put(newKey, tuple[1][oldKey]) + } + tuple[1].keySet().removeAll(workflowArgs.renameKeys.collect{ newKey, oldKey -> oldKey }) + } + tuple + } + + + def chRun = null + def chPassthrough = null + if (workflowArgs.runIf) { + def runIfBranch = chModified.branch{ tup -> + run: workflowArgs.runIf(tup[0], tup[1]) + passthrough: true + } + chRun = runIfBranch.run + chPassthrough = runIfBranch.passthrough + } else { + chRun = chModified + chPassthrough = Channel.empty() + } + + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + + def chArgs = workflowArgs.fromState ? + chRunFiltered | map{ + def new_data = workflowArgs.fromState(it.take(2)) + [it[0], new_data] + } : + chRunFiltered | map {tup -> tup.take(2)} + + // fill in defaults + def chArgsWithDefaults = chArgs + | map { tuple -> + def id_ = tuple[0] + def data_ = tuple[1] + + // TODO: could move fromState to here + + // fetch default params from functionality + def defaultArgs = meta.config.allArguments + .findAll { it.containsKey("default") } + .collectEntries { [ it.plainName, it.default ] } + + // fetch overrides in params + def paramArgs = meta.config.allArguments + .findAll { par -> + def argKey = key_ + "__" + par.plainName + params.containsKey(argKey) + } + .collectEntries { [ it.plainName, params[key_ + "__" + it.plainName] ] } + + // fetch overrides in data + def dataArgs = meta.config.allArguments + .findAll { data_.containsKey(it.plainName) } + .collectEntries { [ it.plainName, data_[it.plainName] ] } + + // combine params + def combinedArgs = defaultArgs + paramArgs + workflowArgs.args + dataArgs + + // remove arguments with explicit null values + combinedArgs + .removeAll{_, val -> val == null || val == "viash_no_value" || val == "force_null"} + + combinedArgs = _processInputValues(combinedArgs, meta.config, id_, key_) + + [id_, combinedArgs] + tuple.drop(2) + } + + // TODO: move some of the _meta.join_id wrangling to the safeJoin() function. + def chInitialOutput = chArgsWithDefaults + | _debug(workflowArgs, "processed") + // run workflow + | innerWorkflowFactory(workflowArgs) + // check output tuple + | map { id_, output_ -> + + // see if output map contains metadata + def meta_ = + output_ instanceof Map && output_.containsKey("_meta") ? + output_["_meta"] : + [:] + def join_id = meta_.join_id ?: id_ + + // remove metadata + output_ = output_.findAll{k, v -> k != "_meta"} + + // check value types + output_ = _processOutputValues(output_, meta.config, id_, key_) + + // simplify output if need be + if (workflowArgs.auto.simplifyOutput && output_.size() == 1) { + output_ = output_.values()[0] + } + + [join_id, id_, output_] + } + // | view{"chInitialOutput: ${it.take(3)}"} + + // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) + // input tuple format: [join_id, id, output, prev_state, ...] + // output tuple format: [join_id, id, new_state, ...] + | map{ tup -> + def new_state = workflowArgs.toState(tup.drop(1).take(3)) + tup.take(2) + [new_state] + tup.drop(4) + } + + if (workflowArgs.auto.publish == "state") { + def chPublish = chNewState + // input tuple format: [join_id, id, new_state, ...] + // output tuple format: [join_id, id, new_state] + | map{ tup -> + tup.take(3) + } + + safeJoin(chPublish, chArgsWithDefaults, key_) + // input tuple format: [join_id, id, new_state, orig_state, ...] + // output tuple format: [id, new_state, orig_state] + | map { tup -> + tup.drop(1).take(3) + } + | publishStatesByConfig(key: key_, config: meta.config) + } + + // remove join_id and meta + chReturn = chNewState + | map { tup -> + // input tuple format: [join_id, id, new_state, ...] + // output tuple format: [id, new_state, ...] + tup.drop(1) + } + | _debug(workflowArgs, "output") + | concat(chPassthrough) + + emit: chReturn + } + + def wf = workflowInstance.cloneWithName(key_) + + // add factory function + wf.metaClass.run = { runArgs -> + workflowFactory(runArgs, workflowArgs, meta) + } + // add config to module for later introspection + wf.metaClass.config = meta.config + + return wf +} + +nextflow.enable.dsl=2 + +// START COMPONENT-SPECIFIC CODE + +// create meta object +meta = [ + "resources_dir": moduleDir.toRealPath().normalize(), + "config": processConfig(readJsonBlob('''{ + "name" : "rsem_prepare_reference", + "namespace" : "rsem", + "version" : "main", + "authors" : [ + { + "name" : "Sai Nirmayi Yasa", + "roles" : [ + "author", + "maintainer" + ], + "info" : { + "links" : { + "email" : "nirmayi@data-intuitive.com", + "github" : "sainirmayi", + "linkedin" : "sai-nirmayi-yasa" + }, + "organizations" : [ + { + "name" : "Data Intuitive", + "href" : "https://www.data-intuitive.com", + "role" : "Junior Bioinformatics Researcher" + } + ] + } + } + ], + "argument_groups" : [ + { + "name" : "Inputs", + "arguments" : [ + { + "type" : "file", + "name" : "--reference_fasta_files", + "description" : "Semi-colon separated list of Multi-FASTA formatted files OR a directory name. If a directory name is specified, RSEM will read all files with suffix \\".fa\\" or \\".fasta\\" in this directory. The files should contain either the sequences of transcripts or an entire genome, depending on whether the '--gtf' option is used.\n", + "example" : [ + "read1.fasta" + ], + "must_exist" : true, + "create_parent" : true, + "required" : true, + "direction" : "input", + "multiple" : true, + "multiple_sep" : ";" + }, + { + "type" : "string", + "name" : "--reference_name", + "description" : "The name of the reference used. RSEM will generate several reference-related files that are prefixed by this name. This name can contain path information (e.g. '/ref/mm9').\n", + "example" : [ + "/ref/mm9" + ], + "required" : true, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + } + ] + }, + { + "name" : "Outputs", + "arguments" : [ + { + "type" : "file", + "name" : "--output", + "description" : "Directory containing reference files generated by RSEM.", + "must_exist" : true, + "create_parent" : true, + "required" : true, + "direction" : "output", + "multiple" : false, + "multiple_sep" : ";" + } + ] + }, + { + "name" : "Other options", + "arguments" : [ + { + "type" : "file", + "name" : "--gtf", + "description" : "Assume that 'reference_fasta_files' contains the sequence of a genome, and extract transcript reference sequences using the gene annotations specified in the GTF file. If this and '--gff3' options are not provided, RSEM will assume 'reference_fasta_files' contains the reference transcripts. In this case, RSEM assumes that name of each sequence in the Multi-FASTA files is its transcript_id.", + "example" : [ + "annotations.gtf" + ], + "must_exist" : true, + "create_parent" : true, + "required" : false, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + }, + { + "type" : "file", + "name" : "--gff3", + "description" : "GFF3 annotation file. Converted to GTF format with the file name 'reference_name.gtf'. Please make sure that 'reference_name.gtf' does not exist.", + "example" : [ + "annotations.gff" + ], + "must_exist" : true, + "create_parent" : true, + "required" : false, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + }, + { + "type" : "string", + "name" : "--gff3_rna_patterns", + "description" : "List of transcript categories (separated by semi-colon). Only transcripts that match the string will be extracted.", + "example" : [ + "mRNA;rRNA" + ], + "required" : false, + "direction" : "input", + "multiple" : true, + "multiple_sep" : ";" + }, + { + "type" : "boolean_true", + "name" : "--gff3_genes_as_transcripts", + "description" : "This option is designed for untypical organisms, such as viruses, whose GFF3 files only contain genes. RSEM will assume each gene as a unique transcript when it converts the GFF3 file into GTF format.", + "direction" : "input" + }, + { + "type" : "string", + "name" : "--trusted_sources", + "description" : "List of trusted sources (separated by semi-colon). Only transcripts coming from these sources will be extracted. If this option is off, all sources are accepted.", + "example" : [ + "ENSEMBL;HAVANA" + ], + "required" : false, + "direction" : "input", + "multiple" : true, + "multiple_sep" : ";" + }, + { + "type" : "file", + "name" : "--transcript_to_gene_map", + "description" : "Use information from this file to map from transcript (isoform) ids to gene ids. Each line of this file should be of the form: \n gene_id transcript_id\nwith the two fields separated by a tab character.\nIf you are using a GTF file for the \\"UCSC Genes\\" gene set from the UCSC Genome Browser, then the \\"knownIsoforms.txt\\" file (obtained from the \\"Downloads\\" section of the UCSC Genome Browser site) is of this format. \nIf this option is off, then the mapping of isoforms to genes depends on whether the '--gtf' option is specified. If '--gtf' is specified, then RSEM uses the \\"gene_id\\" and \\"transcript_id\\" attributes in the GTF file. Otherwise, RSEM assumes that each sequence in the reference sequence files is a separate gene.\n", + "example" : [ + "isoforms.txt" + ], + "must_exist" : true, + "create_parent" : true, + "required" : false, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + }, + { + "type" : "file", + "name" : "--allele_to_gene_map", + "description" : "Use information from to provide gene_id and transcript_id information for each allele-specific transcript. Each line of should be of the form:\n gene_id transcript_id allele_id\nwith the fields separated by a tab character.\nThis option is designed for quantifying allele-specific expression. It is only valid if '--gtf' option is not specified. allele_id should be the sequence names presented in the Multi-FASTA-formatted files.\n", + "must_exist" : true, + "create_parent" : true, + "required" : false, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + }, + { + "type" : "boolean_true", + "name" : "--polyA", + "description" : "Add poly(A) tails to the end of all reference isoforms. The length of poly(A) tail added is specified by '--polyA-length' option. STAR aligner users may not want to use this option.", + "direction" : "input" + }, + { + "type" : "integer", + "name" : "--polyA_length", + "description" : "The length of the poly(A) tails to be added.", + "example" : [ + 125 + ], + "required" : false, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + }, + { + "type" : "file", + "name" : "--no_polyA_subset", + "description" : "Only meaningful if '--polyA' is specified. Do not add poly(A) tails to those transcripts listed in this file containing a list of transcript_ids.", + "example" : [ + "transcript_ids.txt" + ], + "must_exist" : true, + "create_parent" : true, + "required" : false, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + }, + { + "type" : "boolean_true", + "name" : "--bowtie", + "description" : "Build Bowtie indices.", + "direction" : "input" + }, + { + "type" : "boolean_true", + "name" : "--bowtie2", + "description" : "Build Bowtie 2 indices.", + "direction" : "input" + }, + { + "type" : "boolean_true", + "name" : "--star", + "description" : "Build STAR indices.", + "direction" : "input" + }, + { + "type" : "integer", + "name" : "--star_sjdboverhang", + "description" : "Length of the genomic sequence around annotated junction. It is only used for STAR to build splice junctions database and not needed for Bowtie or Bowtie2. It will be passed as the --sjdbOverhang option to STAR. According to STAR's manual, its ideal value is max(ReadLength)-1, e.g. for 2x101 paired-end reads, the ideal value is 101-1=100. In most cases, the default value of 100 will work as well as the ideal value. (Default is 100)", + "example" : [ + 100 + ], + "required" : false, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + }, + { + "type" : "boolean_true", + "name" : "--hisat2_hca", + "description" : "Build HISAT2 indices on the transcriptome according to Human Cell Atlas (HCA) SMART-Seq2 pipeline.", + "direction" : "input" + }, + { + "type" : "boolean_true", + "name" : "--quiet", + "alternatives" : [ + "-q" + ], + "description" : "Suppress the output of logging information.", + "direction" : "input" + } + ] + }, + { + "name" : "Prior-enhanced RSEM options", + "arguments" : [ + { + "type" : "boolean_true", + "name" : "--prep_pRSEM", + "description" : "A Boolean indicating whether to prepare reference files for pRSEM, including building Bowtie indices for a genome and selecting training set isoforms. The index files will be used for aligning ChIP-seq reads in prior-enhanced RSEM and the training set isoforms will be used for learning prior. A path to Bowtie executables and a mappability file in bigWig format are required when this option is on. Currently, Bowtie2 is not supported for prior-enhanced RSEM.", + "direction" : "input" + }, + { + "type" : "file", + "name" : "--mappability_bigwig_file", + "description" : "Full path to a whole-genome mappability file in bigWig format. This file is required for running prior-enhanced RSEM. It is used for selecting a training set of isoforms for prior-learning. This file can be either downloaded from UCSC Genome Browser or generated by GEM (Derrien et al., 2012, PLoS One).", + "must_exist" : true, + "create_parent" : true, + "required" : false, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + } + ] + } + ], + "resources" : [ + { + "type" : "bash_script", + "path" : "script.sh", + "is_executable" : true + } + ], + "description" : "RSEM is a software package for estimating gene and isoform expression levels from RNA-Seq data. This component prepares transcript references for RSEM.\n", + "test_resources" : [ + { + "type" : "bash_script", + "path" : "test.sh", + "is_executable" : true + } + ], + "status" : "enabled", + "requirements" : { + "commands" : [ + "ps" + ] + }, + "keywords" : [ + "Transcriptome", + "Index" + ], + "license" : "GPL-3.0", + "references" : { + "doi" : [ + "10.1186/1471-2105-12-323" + ] + }, + "links" : { + "repository" : "https://github.com/deweylab/RSEM", + "homepage" : "http://deweylab.github.io/RSEM", + "documentation" : "https://deweylab.github.io/RSEM/rsem-prepare-reference.html" + }, + "runners" : [ + { + "type" : "executable", + "id" : "executable", + "docker_setup_strategy" : "ifneedbepullelsecachedbuild" + }, + { + "type" : "nextflow", + "id" : "nextflow", + "directives" : { + "tag" : "$id" + }, + "auto" : { + "simplifyInput" : true, + "simplifyOutput" : false, + "transcript" : false, + "publish" : false + }, + "config" : { + "labels" : { + "mem1gb" : "memory = 1000000000.B", + "mem2gb" : "memory = 2000000000.B", + "mem5gb" : "memory = 5000000000.B", + "mem10gb" : "memory = 10000000000.B", + "mem20gb" : "memory = 20000000000.B", + "mem50gb" : "memory = 50000000000.B", + "mem100gb" : "memory = 100000000000.B", + "mem200gb" : "memory = 200000000000.B", + "mem500gb" : "memory = 500000000000.B", + "mem1tb" : "memory = 1000000000000.B", + "mem2tb" : "memory = 2000000000000.B", + "mem5tb" : "memory = 5000000000000.B", + "mem10tb" : "memory = 10000000000000.B", + "mem20tb" : "memory = 20000000000000.B", + "mem50tb" : "memory = 50000000000000.B", + "mem100tb" : "memory = 100000000000000.B", + "mem200tb" : "memory = 200000000000000.B", + "mem500tb" : "memory = 500000000000000.B", + "mem1gib" : "memory = 1073741824.B", + "mem2gib" : "memory = 2147483648.B", + "mem4gib" : "memory = 4294967296.B", + "mem8gib" : "memory = 8589934592.B", + "mem16gib" : "memory = 17179869184.B", + "mem32gib" : "memory = 34359738368.B", + "mem64gib" : "memory = 68719476736.B", + "mem128gib" : "memory = 137438953472.B", + "mem256gib" : "memory = 274877906944.B", + "mem512gib" : "memory = 549755813888.B", + "mem1tib" : "memory = 1099511627776.B", + "mem2tib" : "memory = 2199023255552.B", + "mem4tib" : "memory = 4398046511104.B", + "mem8tib" : "memory = 8796093022208.B", + "mem16tib" : "memory = 17592186044416.B", + "mem32tib" : "memory = 35184372088832.B", + "mem64tib" : "memory = 70368744177664.B", + "mem128tib" : "memory = 140737488355328.B", + "mem256tib" : "memory = 281474976710656.B", + "mem512tib" : "memory = 562949953421312.B", + "cpu1" : "cpus = 1", + "cpu2" : "cpus = 2", + "cpu5" : "cpus = 5", + "cpu10" : "cpus = 10", + "cpu20" : "cpus = 20", + "cpu50" : "cpus = 50", + "cpu100" : "cpus = 100", + "cpu200" : "cpus = 200", + "cpu500" : "cpus = 500", + "cpu1000" : "cpus = 1000" + } + }, + "debug" : false, + "container" : "docker" + } + ], + "engines" : [ + { + "type" : "docker", + "id" : "docker", + "image" : "ubuntu:22.04", + "target_registry" : "images.viash-hub.com", + "target_tag" : "main", + "namespace_separator" : "/", + "setup" : [ + { + "type" : "apt", + "packages" : [ + "build-essential", + "gcc", + "g++", + "make", + "wget", + "zlib1g-dev", + "unzip xxd", + "perl", + "r-base", + "bowtie2", + "pip", + "git" + ], + "interactive" : false + }, + { + "type" : "python", + "user" : false, + "packages" : [ + "bowtie" + ], + "upgrade" : true + }, + { + "type" : "docker", + "run" : [ + "ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone && \\\\\ncd /tmp && \\\\\nwget --no-check-certificate https://github.com/alexdobin/STAR/archive/refs/tags/${STAR_VERSION}.zip && \\\\\nunzip ${STAR_VERSION}.zip && \\\\\ncd STAR-${STAR_VERSION}/source && \\\\\nmake STARstatic CXXFLAGS_SIMD=-std=c++11 && \\\\\ncp STAR /usr/local/bin && \\\\\ncd /tmp && \\\\\nwget --no-check-certificate https://github.com/deweylab/RSEM/archive/refs/tags/v${RSEM_VERSION}.zip && \\\\\nunzip v${RSEM_VERSION}.zip && \\\\\ncd RSEM-${RSEM_VERSION} && \\\\\nmake && \\\\\nmake install && \\\\\ncd /tmp && \\\\\nwget --no-check-certificate -O bowtie-${BOWTIE_VERSION}-linux-x86_64.zip https://sourceforge.net/projects/bowtie-bio/files/bowtie/${BOWTIE_VERSION}/bowtie-${BOWTIE_VERSION}-linux-x86_64.zip/download && \\\\\nunzip bowtie-${BOWTIE_VERSION}-linux-x86_64.zip && \\\\\ncp bowtie-${BOWTIE_VERSION}-linux-x86_64/bowtie* /usr/local/bin && \\\\\ncd /tmp && \\\\\ngit clone https://github.com/DaehwanKimLab/hisat2.git /tmp/hisat2 && \\\\\ncd /tmp/hisat2 && \\\\\nmake && \\\\\ncp -r hisat2* /usr/local/bin && \\\\\ncd && \\\\\nrm -rf /tmp/STAR-${STAR_VERSION} /tmp/${STAR_VERSION}.zip /tmp/bowtie-${BOWTIE_VERSION}-linux-x86_64 /tmp/hisat2 && \\\\\napt-get --purge autoremove -y ${PACKAGES} && \\\\\napt-get clean \n" + ], + "env" : [ + "STAR_VERSION=2.7.11b", + "RSEM_VERSION=1.3.3", + "BOWTIE_VERSION=1.3.1", + "TZ=Europe/Brussels" + ] + }, + { + "type" : "docker", + "run" : [ + "echo \\"RSEM: `rsem-calculate-expression --version | sed -e 's/Current version: RSEM v//g'`\\" > /var/software_versions.txt && \\\\\necho \\"STAR: `STAR --version`\\" >> /var/software_versions.txt && \\\\\necho \\"bowtie2: `bowtie2 --version | grep -oP '\\\\d+\\\\.\\\\d+\\\\.\\\\d+'`\\" >> /var/software_versions.txt && \\\\\necho \\"bowtie: `bowtie --version | grep -oP 'bowtie-align-s version \\\\K\\\\d+\\\\.\\\\d+\\\\.\\\\d+'`\\" >> /var/software_versions.txt && \\\\\necho \\"HISAT2: `hisat2 --version | grep -oP 'hisat2-align-s version \\\\K\\\\d+\\\\.\\\\d+\\\\.\\\\d+'`\\" >> /var/software_versions.txt\n" + ] + } + ] + }, + { + "type" : "native", + "id" : "native" + } + ], + "build_info" : { + "config" : "/workdir/root/repo/src/rsem/rsem_prepare_reference/config.vsh.yaml", + "runner" : "nextflow", + "engine" : "docker|native", + "output" : "target/nextflow/rsem/rsem_prepare_reference", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", + "git_remote" : "https://github.com/viash-hub/biobox" + }, + "package_config" : { + "name" : "biobox", + "version" : "main", + "description" : "A collection of bioinformatics tools for working with sequence data.\n", + "viash_version" : "0.9.0-RC7", + "source" : "src", + "target" : "target", + "config_mods" : [ + ".requirements.commands := ['ps']\n", + ".engines += { type: \\"native\\" }", + ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", + ".engines[.type == 'docker'].target_tag := 'main'" + ], + "keywords" : [ + "bioinformatics", + "modules", + "sequencing" + ], + "license" : "MIT", + "organization" : "vsh", + "links" : { + "repository" : "https://github.com/viash-hub/biobox", + "issue_tracker" : "https://github.com/viash-hub/biobox/issues" + } + } +}''')) +] + +// resolve dependencies dependencies (if any) + + +// inner workflow +// inner workflow hook +def innerWorkflowFactory(args) { + def rawScript = '''set -e +tempscript=".viash_script.sh" +cat > "$tempscript" << VIASHMAIN +## VIASH START +# The following code has been auto-generated by Viash. +$( if [ ! -z ${VIASH_PAR_REFERENCE_FASTA_FILES+x} ]; then echo "${VIASH_PAR_REFERENCE_FASTA_FILES}" | sed "s#'#'\\"'\\"'#g;s#.*#par_reference_fasta_files='&'#" ; else echo "# par_reference_fasta_files="; fi ) +$( if [ ! -z ${VIASH_PAR_REFERENCE_NAME+x} ]; then echo "${VIASH_PAR_REFERENCE_NAME}" | sed "s#'#'\\"'\\"'#g;s#.*#par_reference_name='&'#" ; else echo "# par_reference_name="; fi ) +$( if [ ! -z ${VIASH_PAR_OUTPUT+x} ]; then echo "${VIASH_PAR_OUTPUT}" | sed "s#'#'\\"'\\"'#g;s#.*#par_output='&'#" ; else echo "# par_output="; fi ) +$( if [ ! -z ${VIASH_PAR_GTF+x} ]; then echo "${VIASH_PAR_GTF}" | sed "s#'#'\\"'\\"'#g;s#.*#par_gtf='&'#" ; else echo "# par_gtf="; fi ) +$( if [ ! -z ${VIASH_PAR_GFF3+x} ]; then echo "${VIASH_PAR_GFF3}" | sed "s#'#'\\"'\\"'#g;s#.*#par_gff3='&'#" ; else echo "# par_gff3="; fi ) +$( if [ ! -z ${VIASH_PAR_GFF3_RNA_PATTERNS+x} ]; then echo "${VIASH_PAR_GFF3_RNA_PATTERNS}" | sed "s#'#'\\"'\\"'#g;s#.*#par_gff3_rna_patterns='&'#" ; else echo "# par_gff3_rna_patterns="; fi ) +$( if [ ! -z ${VIASH_PAR_GFF3_GENES_AS_TRANSCRIPTS+x} ]; then echo "${VIASH_PAR_GFF3_GENES_AS_TRANSCRIPTS}" | sed "s#'#'\\"'\\"'#g;s#.*#par_gff3_genes_as_transcripts='&'#" ; else echo "# par_gff3_genes_as_transcripts="; fi ) +$( if [ ! -z ${VIASH_PAR_TRUSTED_SOURCES+x} ]; then echo "${VIASH_PAR_TRUSTED_SOURCES}" | sed "s#'#'\\"'\\"'#g;s#.*#par_trusted_sources='&'#" ; else echo "# par_trusted_sources="; fi ) +$( if [ ! -z ${VIASH_PAR_TRANSCRIPT_TO_GENE_MAP+x} ]; then echo "${VIASH_PAR_TRANSCRIPT_TO_GENE_MAP}" | sed "s#'#'\\"'\\"'#g;s#.*#par_transcript_to_gene_map='&'#" ; else echo "# par_transcript_to_gene_map="; fi ) +$( if [ ! -z ${VIASH_PAR_ALLELE_TO_GENE_MAP+x} ]; then echo "${VIASH_PAR_ALLELE_TO_GENE_MAP}" | sed "s#'#'\\"'\\"'#g;s#.*#par_allele_to_gene_map='&'#" ; else echo "# par_allele_to_gene_map="; fi ) +$( if [ ! -z ${VIASH_PAR_POLYA+x} ]; then echo "${VIASH_PAR_POLYA}" | sed "s#'#'\\"'\\"'#g;s#.*#par_polyA='&'#" ; else echo "# par_polyA="; fi ) +$( if [ ! -z ${VIASH_PAR_POLYA_LENGTH+x} ]; then echo "${VIASH_PAR_POLYA_LENGTH}" | sed "s#'#'\\"'\\"'#g;s#.*#par_polyA_length='&'#" ; else echo "# par_polyA_length="; fi ) +$( if [ ! -z ${VIASH_PAR_NO_POLYA_SUBSET+x} ]; then echo "${VIASH_PAR_NO_POLYA_SUBSET}" | sed "s#'#'\\"'\\"'#g;s#.*#par_no_polyA_subset='&'#" ; else echo "# par_no_polyA_subset="; fi ) +$( if [ ! -z ${VIASH_PAR_BOWTIE+x} ]; then echo "${VIASH_PAR_BOWTIE}" | sed "s#'#'\\"'\\"'#g;s#.*#par_bowtie='&'#" ; else echo "# par_bowtie="; fi ) +$( if [ ! -z ${VIASH_PAR_BOWTIE2+x} ]; then echo "${VIASH_PAR_BOWTIE2}" | sed "s#'#'\\"'\\"'#g;s#.*#par_bowtie2='&'#" ; else echo "# par_bowtie2="; fi ) +$( if [ ! -z ${VIASH_PAR_STAR+x} ]; then echo "${VIASH_PAR_STAR}" | sed "s#'#'\\"'\\"'#g;s#.*#par_star='&'#" ; else echo "# par_star="; fi ) +$( if [ ! -z ${VIASH_PAR_STAR_SJDBOVERHANG+x} ]; then echo "${VIASH_PAR_STAR_SJDBOVERHANG}" | sed "s#'#'\\"'\\"'#g;s#.*#par_star_sjdboverhang='&'#" ; else echo "# par_star_sjdboverhang="; fi ) +$( if [ ! -z ${VIASH_PAR_HISAT2_HCA+x} ]; then echo "${VIASH_PAR_HISAT2_HCA}" | sed "s#'#'\\"'\\"'#g;s#.*#par_hisat2_hca='&'#" ; else echo "# par_hisat2_hca="; fi ) +$( if [ ! -z ${VIASH_PAR_QUIET+x} ]; then echo "${VIASH_PAR_QUIET}" | sed "s#'#'\\"'\\"'#g;s#.*#par_quiet='&'#" ; else echo "# par_quiet="; fi ) +$( if [ ! -z ${VIASH_PAR_PREP_PRSEM+x} ]; then echo "${VIASH_PAR_PREP_PRSEM}" | sed "s#'#'\\"'\\"'#g;s#.*#par_prep_pRSEM='&'#" ; else echo "# par_prep_pRSEM="; fi ) +$( if [ ! -z ${VIASH_PAR_MAPPABILITY_BIGWIG_FILE+x} ]; then echo "${VIASH_PAR_MAPPABILITY_BIGWIG_FILE}" | sed "s#'#'\\"'\\"'#g;s#.*#par_mappability_bigwig_file='&'#" ; else echo "# par_mappability_bigwig_file="; fi ) +$( if [ ! -z ${VIASH_META_NAME+x} ]; then echo "${VIASH_META_NAME}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_name='&'#" ; else echo "# meta_name="; fi ) +$( if [ ! -z ${VIASH_META_FUNCTIONALITY_NAME+x} ]; then echo "${VIASH_META_FUNCTIONALITY_NAME}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_functionality_name='&'#" ; else echo "# meta_functionality_name="; fi ) +$( if [ ! -z ${VIASH_META_RESOURCES_DIR+x} ]; then echo "${VIASH_META_RESOURCES_DIR}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_resources_dir='&'#" ; else echo "# meta_resources_dir="; fi ) +$( if [ ! -z ${VIASH_META_EXECUTABLE+x} ]; then echo "${VIASH_META_EXECUTABLE}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_executable='&'#" ; else echo "# meta_executable="; fi ) +$( if [ ! -z ${VIASH_META_CONFIG+x} ]; then echo "${VIASH_META_CONFIG}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_config='&'#" ; else echo "# meta_config="; fi ) +$( if [ ! -z ${VIASH_META_TEMP_DIR+x} ]; then echo "${VIASH_META_TEMP_DIR}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_temp_dir='&'#" ; else echo "# meta_temp_dir="; fi ) +$( if [ ! -z ${VIASH_META_CPUS+x} ]; then echo "${VIASH_META_CPUS}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_cpus='&'#" ; else echo "# meta_cpus="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_B+x} ]; then echo "${VIASH_META_MEMORY_B}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_b='&'#" ; else echo "# meta_memory_b="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_KB+x} ]; then echo "${VIASH_META_MEMORY_KB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_kb='&'#" ; else echo "# meta_memory_kb="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_MB+x} ]; then echo "${VIASH_META_MEMORY_MB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_mb='&'#" ; else echo "# meta_memory_mb="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_GB+x} ]; then echo "${VIASH_META_MEMORY_GB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_gb='&'#" ; else echo "# meta_memory_gb="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_TB+x} ]; then echo "${VIASH_META_MEMORY_TB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_tb='&'#" ; else echo "# meta_memory_tb="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_PB+x} ]; then echo "${VIASH_META_MEMORY_PB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_pb='&'#" ; else echo "# meta_memory_pb="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_KIB+x} ]; then echo "${VIASH_META_MEMORY_KIB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_kib='&'#" ; else echo "# meta_memory_kib="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_MIB+x} ]; then echo "${VIASH_META_MEMORY_MIB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_mib='&'#" ; else echo "# meta_memory_mib="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_GIB+x} ]; then echo "${VIASH_META_MEMORY_GIB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_gib='&'#" ; else echo "# meta_memory_gib="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_TIB+x} ]; then echo "${VIASH_META_MEMORY_TIB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_tib='&'#" ; else echo "# meta_memory_tib="; fi ) +$( if [ ! -z ${VIASH_META_MEMORY_PIB+x} ]; then echo "${VIASH_META_MEMORY_PIB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_pib='&'#" ; else echo "# meta_memory_pib="; fi ) + +## VIASH END +#!/bin/bash + +set -eo pipefail + +unset_if_false=( par_gff3_genes_as_transcripts par_polyA par_bowtie par_bowtie2 par_star par_hisat2_hca par_quiet par_prep_pRSEM ) + +for par in \\${unset_if_false[@]}; do + test_val="\\${!par}" + [[ "\\$test_val" == "false" ]] && unset \\$par +done + +# replace ';' with ',' +par_reference_fasta_files=\\$(echo \\$par_reference_fasta_files | tr ';' ',') +par_gff3_rna_patterns=\\$(echo \\$par_gff3_rna_patterns | tr ';' ',') +par_trusted_sources=\\$(echo \\$par_trusted_sources | tr ';' ',') + +echo "\\$par_reference_fasta_files" +rsem-prepare-reference \\\\ + \\${par_gtf:+--gtf "\\${par_gtf}"} \\\\ + \\${par_gff3:+--gff3 "\\${par_gff3}"} \\\\ + \\${par_gff3_rna_patterns:+--gff3-RNA-patterns "\\${par_gff3_rna_patterns}"} \\\\ + \\${par_gff3_genes_as_transcripts:+--gff3-genes-as-transcripts "\\${par_gff3_genes_as_transcripts}"} \\\\ + \\${par_trusted_sources:+--trusted-sources "\\${par_trusted_sources}"} \\\\ + \\${par_transcript_to_gene_map:+--transcript-to-gene-map "\\${par_transcript_to_gene_map}"} \\\\ + \\${par_allele_to_gene_map:+--allele-to-gene-map "\\${par_allele_to_gene_map}"} \\\\ + \\${par_polyA:+--polyA} \\\\ + \\${par_polyA_length:+--polyA-length "\\${par_polyA_length}"} \\\\ + \\${par_no_polyA_subset:+--no-polyA-subset "\\${par_no_polyA_subset}"} \\\\ + \\${par_bowtie:+--bowtie} \\\\ + \\${par_bowtie2:+--bowtie2} \\\\ + \\${par_star:+--star} \\\\ + \\${par_star_sjdboverhang:+--star-sjdboverhang "\\${par_star_sjdboverhang}"} \\\\ + \\${par_hisat2_hca:+--hisat2-hca} \\\\ + \\${par_quiet:+--quiet} \\\\ + \\${par_prep_pRSEM:+--prep-pRSEM} \\\\ + \\${par_mappability_bigwig_file:+--mappability-bigwig-file "\\${par_mappability_bigwig_file}"} \\\\ + \\${meta_cpus:+--num-threads "\\${meta_cpus}"} \\\\ + "\\${par_reference_fasta_files}" \\\\ + "\\${par_reference_name}" + +mkdir -p "\\${par_output}" +mv \\${par_reference_name}.* "\\${par_output}/" +VIASHMAIN +bash "$tempscript" +''' + + return vdsl3WorkflowFactory(args, meta, rawScript) +} + + + +/** + * Generate a workflow for VDSL3 modules. + * + * This function is called by the workflowFactory() function. + * + * Input channel: [id, input_map] + * Output channel: [id, output_map] + * + * Internally, this workflow will convert the input channel + * to a format which the Nextflow module will be able to handle. + */ +def vdsl3WorkflowFactory(Map args, Map meta, String rawScript) { + def key = args["key"] + def processObj = null + + workflow processWf { + take: input_ + main: + + if (processObj == null) { + processObj = _vdsl3ProcessFactory(args, meta, rawScript) + } + + output_ = input_ + | map { tuple -> + def id = tuple[0] + def data_ = tuple[1] + + if (workflow.stubRun) { + // add id if missing + data_ = [id: 'stub'] + data_ + } + + // process input files separately + def inputPaths = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "input" } + .collect { par -> + def val = data_.containsKey(par.plainName) ? data_[par.plainName] : [] + def inputFiles = [] + if (val == null) { + inputFiles = [] + } else if (val instanceof List) { + inputFiles = val + } else if (val instanceof Path) { + inputFiles = [ val ] + } else { + inputFiles = [] + } + if (!workflow.stubRun) { + // throw error when an input file doesn't exist + inputFiles.each{ file -> + assert file.exists() : + "Error in module '${key}' id '${id}' argument '${par.plainName}'.\n" + + " Required input file does not exist.\n" + + " Path: '$file'.\n" + + " Expected input file to exist" + } + } + inputFiles + } + + // remove input files + def argsExclInputFiles = meta.config.allArguments + .findAll { (it.type != "file" || it.direction != "input") && data_.containsKey(it.plainName) } + .collectEntries { par -> + def parName = par.plainName + def val = data_[parName] + if (par.multiple && val instanceof Collection) { + val = val.join(par.multiple_sep) + } + if (par.direction == "output" && par.type == "file") { + val = val.replaceAll('\\$id', id).replaceAll('\\$key', key) + } + [parName, val] + } + + [ id ] + inputPaths + [ argsExclInputFiles, meta.resources_dir ] + } + | processObj + | map { output -> + def outputFiles = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "output" } + .indexed() + .collectEntries{ index, par -> + def out = output[index + 1] + // strip dummy '.exitcode' file from output (see nextflow-io/nextflow#2678) + if (!out instanceof List || out.size() <= 1) { + if (par.multiple) { + out = [] + } else { + assert !par.required : + "Error in module '${key}' id '${output[0]}' argument '${par.plainName}'.\n" + + " Required output file is missing" + out = null + } + } else if (out.size() == 2 && !par.multiple) { + out = out[1] + } else { + out = out.drop(1) + } + [ par.plainName, out ] + } + + // drop null outputs + outputFiles.removeAll{it.value == null} + + [ output[0], outputFiles ] + } + emit: output_ + } + + return processWf +} + +// depends on: session? +def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) { + // autodetect process key + def wfKey = workflowArgs["key"] + def procKeyPrefix = "${wfKey}_process" + def scriptMeta = nextflow.script.ScriptMeta.current() + def existing = scriptMeta.getProcessNames().findAll{it.startsWith(procKeyPrefix)} + def numbers = existing.collect{it.replace(procKeyPrefix, "0").toInteger()} + def newNumber = (numbers + [-1]).max() + 1 + + def procKey = newNumber == 0 ? procKeyPrefix : "$procKeyPrefix$newNumber" + + if (newNumber > 0) { + log.warn "Key for module '${wfKey}' is duplicated.\n", + "If you run a component multiple times in the same workflow,\n" + + "it's recommended you set a unique key for every call,\n" + + "for example: ${wfKey}.run(key: \"foo\")." + } + + // subset directives and convert to list of tuples + def drctv = workflowArgs.directives + + // TODO: unit test the two commands below + // convert publish array into tags + def valueToStr = { val -> + // ignore closures + if (val instanceof CharSequence) { + if (!val.matches('^[{].*[}]$')) { + '"' + val + '"' + } else { + val + } + } else if (val instanceof List) { + "[" + val.collect{valueToStr(it)}.join(", ") + "]" + } else if (val instanceof Map) { + "[" + val.collect{k, v -> k + ": " + valueToStr(v)}.join(", ") + "]" + } else { + val.inspect() + } + } + + // multiple entries allowed: label, publishdir + def drctvStrs = drctv.collect { key, value -> + if (key in ["label", "publishDir"]) { + value.collect{ val -> + if (val instanceof Map) { + "\n$key " + val.collect{ k, v -> k + ": " + valueToStr(v) }.join(", ") + } else if (val == null) { + "" + } else { + "\n$key " + valueToStr(val) + } + }.join() + } else if (value instanceof Map) { + "\n$key " + value.collect{ k, v -> k + ": " + valueToStr(v) }.join(", ") + } else { + "\n$key " + valueToStr(value) + } + }.join() + + def inputPaths = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "input" } + .collect { ', path(viash_par_' + it.plainName + ', stageAs: "_viash_par/' + it.plainName + '_?/*")' } + .join() + + def outputPaths = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "output" } + .collect { par -> + // insert dummy into every output (see nextflow-io/nextflow#2678) + if (!par.multiple) { + ', path{[".exitcode", args.' + par.plainName + ']}' + } else { + ', path{[".exitcode"] + args.' + par.plainName + '}' + } + } + .join() + + // TODO: move this functionality somewhere else? + if (workflowArgs.auto.transcript) { + outputPaths = outputPaths + ', path{[".exitcode", ".command*"]}' + } else { + outputPaths = outputPaths + ', path{[".exitcode"]}' + } + + // create dirs for output files (based on BashWrapper.createParentFiles) + def createParentStr = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "output" && it.create_parent } + .collect { par -> + "\${ args.containsKey(\"${par.plainName}\") ? \"mkdir_parent \\\"\" + (args[\"${par.plainName}\"] instanceof String ? args[\"${par.plainName}\"] : args[\"${par.plainName}\"].join('\" \"')) + \"\\\"\" : \"\" }" + } + .join("\n") + + // construct inputFileExports + def inputFileExports = meta.config.allArguments + .findAll { it.type == "file" && it.direction.toLowerCase() == "input" } + .collect { par -> + def viash_par_contents = "(viash_par_${par.plainName} instanceof List ? viash_par_${par.plainName}.join(\"${par.multiple_sep}\") : viash_par_${par.plainName})" + "\n\${viash_par_${par.plainName}.empty ? \"\" : \"export VIASH_PAR_${par.plainName.toUpperCase()}=\\\"\" + ${viash_par_contents} + \"\\\"\"}" + } + + // NOTE: if using docker, use /tmp instead of tmpDir! + def tmpDir = java.nio.file.Paths.get( + System.getenv('NXF_TEMP') ?: + System.getenv('VIASH_TEMP') ?: + System.getenv('VIASH_TMPDIR') ?: + System.getenv('VIASH_TEMPDIR') ?: + System.getenv('VIASH_TMP') ?: + System.getenv('TEMP') ?: + System.getenv('TMPDIR') ?: + System.getenv('TEMPDIR') ?: + System.getenv('TMP') ?: + '/tmp' + ).toAbsolutePath() + + // construct stub + def stub = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "output" } + .collect { par -> + "\${ args.containsKey(\"${par.plainName}\") ? \"touch2 \\\"\" + (args[\"${par.plainName}\"] instanceof String ? args[\"${par.plainName}\"].replace(\"_*\", \"_0\") : args[\"${par.plainName}\"].join('\" \"')) + \"\\\"\" : \"\" }" + } + .join("\n") + + // escape script + def escapedScript = rawScript.replace('\\', '\\\\').replace('$', '\\$').replace('"""', '\\"\\"\\"') + + // publishdir assert + def assertStr = (workflowArgs.auto.publish == true) || workflowArgs.auto.transcript ? + """\nassert task.publishDir.size() > 0: "if auto.publish is true, params.publish_dir needs to be defined.\\n Example: --publish_dir './output/'" """ : + "" + + // generate process string + def procStr = + """nextflow.enable.dsl=2 + | + |process $procKey {$drctvStrs + |input: + | tuple val(id)$inputPaths, val(args), path(resourcesDir, stageAs: ".viash_meta_resources") + |output: + | tuple val("\$id")$outputPaths, optional: true + |stub: + |\"\"\" + |touch2() { mkdir -p "\\\$(dirname "\\\$1")" && touch "\\\$1" ; } + |$stub + |\"\"\" + |script:$assertStr + |def escapeText = { s -> s.toString().replaceAll('([`"])', '\\\\\\\\\$1') } + |def parInject = args + | .findAll{key, value -> value != null} + | .collect{key, value -> "export VIASH_PAR_\${key.toUpperCase()}=\\\"\${escapeText(value)}\\\""} + | .join("\\n") + |\"\"\" + |# meta exports + |export VIASH_META_RESOURCES_DIR="\${resourcesDir}" + |export VIASH_META_TEMP_DIR="${['docker', 'podman', 'charliecloud'].any{ it == workflow.containerEngine } ? '/tmp' : tmpDir}" + |export VIASH_META_NAME="${meta.config.name}" + |# export VIASH_META_EXECUTABLE="\\\$VIASH_META_RESOURCES_DIR/\\\$VIASH_META_NAME" + |export VIASH_META_CONFIG="\\\$VIASH_META_RESOURCES_DIR/.config.vsh.yaml" + |\${task.cpus ? "export VIASH_META_CPUS=\$task.cpus" : "" } + |\${task.memory?.bytes != null ? "export VIASH_META_MEMORY_B=\$task.memory.bytes" : "" } + |if [ ! -z \\\${VIASH_META_MEMORY_B+x} ]; then + | export VIASH_META_MEMORY_KB=\\\$(( (\\\$VIASH_META_MEMORY_B+999) / 1000 )) + | export VIASH_META_MEMORY_MB=\\\$(( (\\\$VIASH_META_MEMORY_KB+999) / 1000 )) + | export VIASH_META_MEMORY_GB=\\\$(( (\\\$VIASH_META_MEMORY_MB+999) / 1000 )) + | export VIASH_META_MEMORY_TB=\\\$(( (\\\$VIASH_META_MEMORY_GB+999) / 1000 )) + | export VIASH_META_MEMORY_PB=\\\$(( (\\\$VIASH_META_MEMORY_TB+999) / 1000 )) + | export VIASH_META_MEMORY_KIB=\\\$(( (\\\$VIASH_META_MEMORY_B+1023) / 1024 )) + | export VIASH_META_MEMORY_MIB=\\\$(( (\\\$VIASH_META_MEMORY_KIB+1023) / 1024 )) + | export VIASH_META_MEMORY_GIB=\\\$(( (\\\$VIASH_META_MEMORY_MIB+1023) / 1024 )) + | export VIASH_META_MEMORY_TIB=\\\$(( (\\\$VIASH_META_MEMORY_GIB+1023) / 1024 )) + | export VIASH_META_MEMORY_PIB=\\\$(( (\\\$VIASH_META_MEMORY_TIB+1023) / 1024 )) + |fi + | + |# meta synonyms + |export VIASH_TEMP="\\\$VIASH_META_TEMP_DIR" + |export TEMP_DIR="\\\$VIASH_META_TEMP_DIR" + | + |# create output dirs if need be + |function mkdir_parent { + | for file in "\\\$@"; do + | mkdir -p "\\\$(dirname "\\\$file")" + | done + |} + |$createParentStr + | + |# argument exports${inputFileExports.join()} + |\$parInject + | + |# process script + |${escapedScript} + |\"\"\" + |} + |""".stripMargin() + + // TODO: print on debug + // if (workflowArgs.debug == true) { + // println("######################\n$procStr\n######################") + // } + + // write process to temp file + def tempFile = java.nio.file.Files.createTempFile("viash-process-${procKey}-", ".nf") + addShutdownHook { java.nio.file.Files.deleteIfExists(tempFile) } + tempFile.text = procStr + + // create process from temp file + def binding = new nextflow.script.ScriptBinding([:]) + def session = nextflow.Nextflow.getSession() + def parser = new nextflow.script.ScriptParser(session) + .setModule(true) + .setBinding(binding) + def moduleScript = parser.runScript(tempFile) + .getScript() + + // register module in meta + def module = new nextflow.script.IncludeDef.Module(name: procKey) + scriptMeta.addModule(moduleScript, module.name, module.alias) + + // retrieve and return process from meta + return scriptMeta.getProcess(procKey) +} + +// defaults +meta["defaults"] = [ + // key to be used to trace the process and determine output names + key: null, + + // fixed arguments to be passed to script + args: [:], + + // default directives + directives: readJsonBlob('''{ + "container" : { + "registry" : "images.viash-hub.com", + "image" : "vsh/biobox/rsem/rsem_prepare_reference", + "tag" : "main" + }, + "tag" : "$id" +}'''), + + // auto settings + auto: readJsonBlob('''{ + "simplifyInput" : true, + "simplifyOutput" : false, + "transcript" : false, + "publish" : false +}'''), + + // Apply a map over the incoming tuple + // Example: `{ tup -> [ tup[0], [input: tup[1].output] ] + tup.drop(2) }` + map: null, + + // Apply a map over the ID element of a tuple (i.e. the first element) + // Example: `{ id -> id + "_foo" }` + mapId: null, + + // Apply a map over the data element of a tuple (i.e. the second element) + // Example: `{ data -> [ input: data.output ] }` + mapData: null, + + // Apply a map over the passthrough elements of a tuple (i.e. the tuple excl. the first two elements) + // Example: `{ pt -> pt.drop(1) }` + mapPassthrough: null, + + // Filter the channel + // Example: `{ tup -> tup[0] == "foo" }` + filter: null, + + // Choose whether or not to run the component on the tuple if the condition is true. + // Otherwise, the tuple will be passed through. + // Example: `{ tup -> tup[0] != "skip_this" }` + runIf: null, + + // Rename keys in the data field of the tuple (i.e. the second element) + // Will likely be deprecated in favour of `fromState`. + // Example: `[ "new_key": "old_key" ]` + renameKeys: null, + + // Fetch data from the state and pass it to the module without altering the current state. + // + // `fromState` should be `null`, `List[String]`, `Map[String, String]` or a function. + // + // - If it is `null`, the state will be passed to the module as is. + // - If it is a `List[String]`, the data will be the values of the state at the given keys. + // - If it is a `Map[String, String]`, the data will be the values of the state at the given keys, with the keys renamed according to the map. + // - If it is a function, the tuple (`[id, state]`) in the channel will be passed to the function, and the result will be used as the data. + // + // Example: `{ id, state -> [input: state.fastq_file] }` + // Default: `null` + fromState: null, + + // Determine how the state should be updated after the module has been run. + // + // `toState` should be `null`, `List[String]`, `Map[String, String]` or a function. + // + // - If it is `null`, the state will be replaced with the output of the module. + // - If it is a `List[String]`, the state will be updated with the values of the data at the given keys. + // - If it is a `Map[String, String]`, the state will be updated with the values of the data at the given keys, with the keys renamed according to the map. + // - If it is a function, a tuple (`[id, output, state]`) will be passed to the function, and the result will be used as the new state. + // + // Example: `{ id, output, state -> state + [counts: state.output] }` + // Default: `{ id, output, state -> output }` + toState: null, + + // Whether or not to print debug messages + // Default: `false` + debug: false +] + +// initialise default workflow +meta["workflow"] = workflowFactory([key: meta.config.name], meta.defaults, meta) + +// add workflow to environment +nextflow.script.ScriptMeta.current().addDefinition(meta.workflow) + +// anonymous workflow for running this module as a standalone +workflow { + // add id argument if it's not already in the config + // TODO: deep copy + def newConfig = deepClone(meta.config) + def newParams = deepClone(params) + + def argsContainsId = newConfig.allArguments.any{it.plainName == "id"} + if (!argsContainsId) { + def idArg = [ + 'name': '--id', + 'required': false, + 'type': 'string', + 'description': 'A unique id for every entry.', + 'multiple': false + ] + newConfig.arguments.add(0, idArg) + newConfig = processConfig(newConfig) + } + if (!newParams.containsKey("id")) { + newParams.id = "run" + } + + helpMessage(newConfig) + + channelFromParams(newParams, newConfig) + // make sure id is not in the state if id is not in the args + | map {id, state -> + if (!argsContainsId) { + [id, state.findAll{k, v -> k != "id"}] + } else { + [id, state] + } + } + | meta.workflow.run( + auto: [ publish: "state" ] + ) +} + +// END COMPONENT-SPECIFIC CODE diff --git a/target/nextflow/rsem/rsem_prepare_reference/nextflow.config b/target/nextflow/rsem/rsem_prepare_reference/nextflow.config new file mode 100644 index 00000000..35bb7479 --- /dev/null +++ b/target/nextflow/rsem/rsem_prepare_reference/nextflow.config @@ -0,0 +1,126 @@ +manifest { + name = 'rsem/rsem_prepare_reference' + mainScript = 'main.nf' + nextflowVersion = '!>=20.12.1-edge' + version = 'main' + description = 'RSEM is a software package for estimating gene and isoform expression levels from RNA-Seq data. This component prepares transcript references for RSEM.\n' + author = 'Sai Nirmayi Yasa' +} + +process.container = 'nextflow/bash:latest' + +// detect tempdir +tempDir = java.nio.file.Paths.get( + System.getenv('NXF_TEMP') ?: + System.getenv('VIASH_TEMP') ?: + System.getenv('TEMPDIR') ?: + System.getenv('TMPDIR') ?: + '/tmp' +).toAbsolutePath() + +profiles { + no_publish { + process { + withName: '.*' { + publishDir = [ + enabled: false + ] + } + } + } + mount_temp { + docker.temp = tempDir + podman.temp = tempDir + charliecloud.temp = tempDir + } + docker { + docker.enabled = true + // docker.userEmulation = true + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + singularity { + singularity.enabled = true + singularity.autoMounts = true + docker.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + podman { + podman.enabled = true + docker.enabled = false + singularity.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + shifter { + shifter.enabled = true + docker.enabled = false + singularity.enabled = false + podman.enabled = false + charliecloud.enabled = false + } + charliecloud { + charliecloud.enabled = true + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + } +} + +process{ + withLabel: mem1gb { memory = 1000000000.B } + withLabel: mem2gb { memory = 2000000000.B } + withLabel: mem5gb { memory = 5000000000.B } + withLabel: mem10gb { memory = 10000000000.B } + withLabel: mem20gb { memory = 20000000000.B } + withLabel: mem50gb { memory = 50000000000.B } + withLabel: mem100gb { memory = 100000000000.B } + withLabel: mem200gb { memory = 200000000000.B } + withLabel: mem500gb { memory = 500000000000.B } + withLabel: mem1tb { memory = 1000000000000.B } + withLabel: mem2tb { memory = 2000000000000.B } + withLabel: mem5tb { memory = 5000000000000.B } + withLabel: mem10tb { memory = 10000000000000.B } + withLabel: mem20tb { memory = 20000000000000.B } + withLabel: mem50tb { memory = 50000000000000.B } + withLabel: mem100tb { memory = 100000000000000.B } + withLabel: mem200tb { memory = 200000000000000.B } + withLabel: mem500tb { memory = 500000000000000.B } + withLabel: mem1gib { memory = 1073741824.B } + withLabel: mem2gib { memory = 2147483648.B } + withLabel: mem4gib { memory = 4294967296.B } + withLabel: mem8gib { memory = 8589934592.B } + withLabel: mem16gib { memory = 17179869184.B } + withLabel: mem32gib { memory = 34359738368.B } + withLabel: mem64gib { memory = 68719476736.B } + withLabel: mem128gib { memory = 137438953472.B } + withLabel: mem256gib { memory = 274877906944.B } + withLabel: mem512gib { memory = 549755813888.B } + withLabel: mem1tib { memory = 1099511627776.B } + withLabel: mem2tib { memory = 2199023255552.B } + withLabel: mem4tib { memory = 4398046511104.B } + withLabel: mem8tib { memory = 8796093022208.B } + withLabel: mem16tib { memory = 17592186044416.B } + withLabel: mem32tib { memory = 35184372088832.B } + withLabel: mem64tib { memory = 70368744177664.B } + withLabel: mem128tib { memory = 140737488355328.B } + withLabel: mem256tib { memory = 281474976710656.B } + withLabel: mem512tib { memory = 562949953421312.B } + withLabel: cpu1 { cpus = 1 } + withLabel: cpu2 { cpus = 2 } + withLabel: cpu5 { cpus = 5 } + withLabel: cpu10 { cpus = 10 } + withLabel: cpu20 { cpus = 20 } + withLabel: cpu50 { cpus = 50 } + withLabel: cpu100 { cpus = 100 } + withLabel: cpu200 { cpus = 200 } + withLabel: cpu500 { cpus = 500 } + withLabel: cpu1000 { cpus = 1000 } +} + + diff --git a/target/nextflow/rsem/rsem_prepare_reference/nextflow_schema.json b/target/nextflow/rsem/rsem_prepare_reference/nextflow_schema.json new file mode 100644 index 00000000..14742746 --- /dev/null +++ b/target/nextflow/rsem/rsem_prepare_reference/nextflow_schema.json @@ -0,0 +1,321 @@ +{ +"$schema": "http://json-schema.org/draft-07/schema", +"title": "rsem_prepare_reference", +"description": "RSEM is a software package for estimating gene and isoform expression levels from RNA-Seq data. This component prepares transcript references for RSEM.\n", +"type": "object", +"definitions": { + + + + "inputs" : { + "title": "Inputs", + "type": "object", + "description": "No description", + "properties": { + + + "reference_fasta_files": { + "type": + "string", + "description": "Type: List of `file`, required, example: `read1.fasta`, multiple_sep: `\";\"`. Semi-colon separated list of Multi-FASTA formatted files OR a directory name", + "help_text": "Type: List of `file`, required, example: `read1.fasta`, multiple_sep: `\";\"`. Semi-colon separated list of Multi-FASTA formatted files OR a directory name. If a directory name is specified, RSEM will read all files with suffix \".fa\" or \".fasta\" in this directory. The files should contain either the sequences of transcripts or an entire genome, depending on whether the \u0027--gtf\u0027 option is used.\n" + + } + + + , + "reference_name": { + "type": + "string", + "description": "Type: `string`, required, example: `/ref/mm9`. The name of the reference used", + "help_text": "Type: `string`, required, example: `/ref/mm9`. The name of the reference used. RSEM will generate several reference-related files that are prefixed by this name. This name can contain path information (e.g. \u0027/ref/mm9\u0027).\n" + + } + + +} +}, + + + "outputs" : { + "title": "Outputs", + "type": "object", + "description": "No description", + "properties": { + + + "output": { + "type": + "string", + "description": "Type: `file`, required, default: `$id.$key.output.output`. Directory containing reference files generated by RSEM", + "help_text": "Type: `file`, required, default: `$id.$key.output.output`. Directory containing reference files generated by RSEM." + , + "default": "$id.$key.output.output" + } + + +} +}, + + + "other options" : { + "title": "Other options", + "type": "object", + "description": "No description", + "properties": { + + + "gtf": { + "type": + "string", + "description": "Type: `file`, example: `annotations.gtf`. Assume that \u0027reference_fasta_files\u0027 contains the sequence of a genome, and extract transcript reference sequences using the gene annotations specified in the GTF file", + "help_text": "Type: `file`, example: `annotations.gtf`. Assume that \u0027reference_fasta_files\u0027 contains the sequence of a genome, and extract transcript reference sequences using the gene annotations specified in the GTF file. If this and \u0027--gff3\u0027 options are not provided, RSEM will assume \u0027reference_fasta_files\u0027 contains the reference transcripts. In this case, RSEM assumes that name of each sequence in the Multi-FASTA files is its transcript_id." + + } + + + , + "gff3": { + "type": + "string", + "description": "Type: `file`, example: `annotations.gff`. GFF3 annotation file", + "help_text": "Type: `file`, example: `annotations.gff`. GFF3 annotation file. Converted to GTF format with the file name \u0027reference_name.gtf\u0027. Please make sure that \u0027reference_name.gtf\u0027 does not exist." + + } + + + , + "gff3_rna_patterns": { + "type": + "string", + "description": "Type: List of `string`, example: `mRNA;rRNA`, multiple_sep: `\";\"`. List of transcript categories (separated by semi-colon)", + "help_text": "Type: List of `string`, example: `mRNA;rRNA`, multiple_sep: `\";\"`. List of transcript categories (separated by semi-colon). Only transcripts that match the string will be extracted." + + } + + + , + "gff3_genes_as_transcripts": { + "type": + "boolean", + "description": "Type: `boolean_true`, default: `false`. This option is designed for untypical organisms, such as viruses, whose GFF3 files only contain genes", + "help_text": "Type: `boolean_true`, default: `false`. This option is designed for untypical organisms, such as viruses, whose GFF3 files only contain genes. RSEM will assume each gene as a unique transcript when it converts the GFF3 file into GTF format." + , + "default": "False" + } + + + , + "trusted_sources": { + "type": + "string", + "description": "Type: List of `string`, example: `ENSEMBL;HAVANA`, multiple_sep: `\";\"`. List of trusted sources (separated by semi-colon)", + "help_text": "Type: List of `string`, example: `ENSEMBL;HAVANA`, multiple_sep: `\";\"`. List of trusted sources (separated by semi-colon). Only transcripts coming from these sources will be extracted. If this option is off, all sources are accepted." + + } + + + , + "transcript_to_gene_map": { + "type": + "string", + "description": "Type: `file`, example: `isoforms.txt`. Use information from this file to map from transcript (isoform) ids to gene ids", + "help_text": "Type: `file`, example: `isoforms.txt`. Use information from this file to map from transcript (isoform) ids to gene ids. Each line of this file should be of the form: \n gene_id transcript_id\nwith the two fields separated by a tab character.\nIf you are using a GTF file for the \"UCSC Genes\" gene set from the UCSC Genome Browser, then the \"knownIsoforms.txt\" file (obtained from the \"Downloads\" section of the UCSC Genome Browser site) is of this format. \nIf this option is off, then the mapping of isoforms to genes depends on whether the \u0027--gtf\u0027 option is specified. If \u0027--gtf\u0027 is specified, then RSEM uses the \"gene_id\" and \"transcript_id\" attributes in the GTF file. Otherwise, RSEM assumes that each sequence in the reference sequence files is a separate gene.\n" + + } + + + , + "allele_to_gene_map": { + "type": + "string", + "description": "Type: `file`. Use information from \u003cfile\u003e to provide gene_id and transcript_id information for each allele-specific transcript", + "help_text": "Type: `file`. Use information from \u003cfile\u003e to provide gene_id and transcript_id information for each allele-specific transcript. Each line of \u003cfile\u003e should be of the form:\n gene_id transcript_id allele_id\nwith the fields separated by a tab character.\nThis option is designed for quantifying allele-specific expression. It is only valid if \u0027--gtf\u0027 option is not specified. allele_id should be the sequence names presented in the Multi-FASTA-formatted files.\n" + + } + + + , + "polyA": { + "type": + "boolean", + "description": "Type: `boolean_true`, default: `false`. Add poly(A) tails to the end of all reference isoforms", + "help_text": "Type: `boolean_true`, default: `false`. Add poly(A) tails to the end of all reference isoforms. The length of poly(A) tail added is specified by \u0027--polyA-length\u0027 option. STAR aligner users may not want to use this option." + , + "default": "False" + } + + + , + "polyA_length": { + "type": + "integer", + "description": "Type: `integer`, example: `125`. The length of the poly(A) tails to be added", + "help_text": "Type: `integer`, example: `125`. The length of the poly(A) tails to be added." + + } + + + , + "no_polyA_subset": { + "type": + "string", + "description": "Type: `file`, example: `transcript_ids.txt`. Only meaningful if \u0027--polyA\u0027 is specified", + "help_text": "Type: `file`, example: `transcript_ids.txt`. Only meaningful if \u0027--polyA\u0027 is specified. Do not add poly(A) tails to those transcripts listed in this file containing a list of transcript_ids." + + } + + + , + "bowtie": { + "type": + "boolean", + "description": "Type: `boolean_true`, default: `false`. Build Bowtie indices", + "help_text": "Type: `boolean_true`, default: `false`. Build Bowtie indices." + , + "default": "False" + } + + + , + "bowtie2": { + "type": + "boolean", + "description": "Type: `boolean_true`, default: `false`. Build Bowtie 2 indices", + "help_text": "Type: `boolean_true`, default: `false`. Build Bowtie 2 indices." + , + "default": "False" + } + + + , + "star": { + "type": + "boolean", + "description": "Type: `boolean_true`, default: `false`. Build STAR indices", + "help_text": "Type: `boolean_true`, default: `false`. Build STAR indices." + , + "default": "False" + } + + + , + "star_sjdboverhang": { + "type": + "integer", + "description": "Type: `integer`, example: `100`. Length of the genomic sequence around annotated junction", + "help_text": "Type: `integer`, example: `100`. Length of the genomic sequence around annotated junction. It is only used for STAR to build splice junctions database and not needed for Bowtie or Bowtie2. It will be passed as the --sjdbOverhang option to STAR. According to STAR\u0027s manual, its ideal value is max(ReadLength)-1, e.g. for 2x101 paired-end reads, the ideal value is 101-1=100. In most cases, the default value of 100 will work as well as the ideal value. (Default is 100)" + + } + + + , + "hisat2_hca": { + "type": + "boolean", + "description": "Type: `boolean_true`, default: `false`. Build HISAT2 indices on the transcriptome according to Human Cell Atlas (HCA) SMART-Seq2 pipeline", + "help_text": "Type: `boolean_true`, default: `false`. Build HISAT2 indices on the transcriptome according to Human Cell Atlas (HCA) SMART-Seq2 pipeline." + , + "default": "False" + } + + + , + "quiet": { + "type": + "boolean", + "description": "Type: `boolean_true`, default: `false`. Suppress the output of logging information", + "help_text": "Type: `boolean_true`, default: `false`. Suppress the output of logging information." + , + "default": "False" + } + + +} +}, + + + "prior-enhanced rsem options" : { + "title": "Prior-enhanced RSEM options", + "type": "object", + "description": "No description", + "properties": { + + + "prep_pRSEM": { + "type": + "boolean", + "description": "Type: `boolean_true`, default: `false`. A Boolean indicating whether to prepare reference files for pRSEM, including building Bowtie indices for a genome and selecting training set isoforms", + "help_text": "Type: `boolean_true`, default: `false`. A Boolean indicating whether to prepare reference files for pRSEM, including building Bowtie indices for a genome and selecting training set isoforms. The index files will be used for aligning ChIP-seq reads in prior-enhanced RSEM and the training set isoforms will be used for learning prior. A path to Bowtie executables and a mappability file in bigWig format are required when this option is on. Currently, Bowtie2 is not supported for prior-enhanced RSEM." + , + "default": "False" + } + + + , + "mappability_bigwig_file": { + "type": + "string", + "description": "Type: `file`. Full path to a whole-genome mappability file in bigWig format", + "help_text": "Type: `file`. Full path to a whole-genome mappability file in bigWig format. This file is required for running prior-enhanced RSEM. It is used for selecting a training set of isoforms for prior-learning. This file can be either downloaded from UCSC Genome Browser or generated by GEM (Derrien et al., 2012, PLoS One)." + + } + + +} +}, + + + "nextflow input-output arguments" : { + "title": "Nextflow input-output arguments", + "type": "object", + "description": "Input/output parameters for Nextflow itself. Please note that both publishDir and publish_dir are supported but at least one has to be configured.", + "properties": { + + + "publish_dir": { + "type": + "string", + "description": "Type: `string`, required, example: `output/`. Path to an output directory", + "help_text": "Type: `string`, required, example: `output/`. Path to an output directory." + + } + + + , + "param_list": { + "type": + "string", + "description": "Type: `string`, example: `my_params.yaml`. Allows inputting multiple parameter sets to initialise a Nextflow channel", + "help_text": "Type: `string`, example: `my_params.yaml`. Allows inputting multiple parameter sets to initialise a Nextflow channel. A `param_list` can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob.\n\n* A list of maps (as-is) where the keys of each map corresponds to the arguments of the pipeline. Example: in a `nextflow.config` file: `param_list: [ [\u0027id\u0027: \u0027foo\u0027, \u0027input\u0027: \u0027foo.txt\u0027], [\u0027id\u0027: \u0027bar\u0027, \u0027input\u0027: \u0027bar.txt\u0027] ]`.\n* A csv file should have column names which correspond to the different arguments of this pipeline. Example: `--param_list data.csv` with columns `id,input`.\n* A json or a yaml file should be a list of maps, each of which has keys corresponding to the arguments of the pipeline. Example: `--param_list data.json` with contents `[ {\u0027id\u0027: \u0027foo\u0027, \u0027input\u0027: \u0027foo.txt\u0027}, {\u0027id\u0027: \u0027bar\u0027, \u0027input\u0027: \u0027bar.txt\u0027} ]`.\n* A yaml blob can also be passed directly as a string. Example: `--param_list \"[ {\u0027id\u0027: \u0027foo\u0027, \u0027input\u0027: \u0027foo.txt\u0027}, {\u0027id\u0027: \u0027bar\u0027, \u0027input\u0027: \u0027bar.txt\u0027} ]\"`.\n\nWhen passing a csv, json or yaml file, relative path names are relativized to the location of the parameter file. No relativation is performed when `param_list` is a list of maps (as-is) or a yaml blob.", + "hidden": true + + } + + +} +} +}, +"allOf": [ + + { + "$ref": "#/definitions/inputs" + }, + + { + "$ref": "#/definitions/outputs" + }, + + { + "$ref": "#/definitions/other options" + }, + + { + "$ref": "#/definitions/prior-enhanced rsem options" + }, + + { + "$ref": "#/definitions/nextflow input-output arguments" + } +] +} diff --git a/target/nextflow/salmon/salmon_index/.config.vsh.yaml b/target/nextflow/salmon/salmon_index/.config.vsh.yaml index 7f9b4ff5..f999ff42 100644 --- a/target/nextflow/salmon/salmon_index/.config.vsh.yaml +++ b/target/nextflow/salmon/salmon_index/.config.vsh.yaml @@ -276,15 +276,15 @@ build_info: engine: "docker|native" output: "target/nextflow/salmon/salmon_index" executable: "target/nextflow/salmon/salmon_index/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/salmon/salmon_index/main.nf b/target/nextflow/salmon/salmon_index/main.nf index 258d4cb5..73b13110 100644 --- a/target/nextflow/salmon/salmon_index/main.nf +++ b/target/nextflow/salmon/salmon_index/main.nf @@ -1,6 +1,6 @@ // salmon_index main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -763,8 +763,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -786,7 +789,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -824,8 +840,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1601,8 +1620,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2629,30 +2648,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2723,7 +2743,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -3102,15 +3122,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/salmon/salmon_index", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/salmon/salmon_quant/.config.vsh.yaml b/target/nextflow/salmon/salmon_quant/.config.vsh.yaml index 55af9657..d62cf3df 100644 --- a/target/nextflow/salmon/salmon_quant/.config.vsh.yaml +++ b/target/nextflow/salmon/salmon_quant/.config.vsh.yaml @@ -1172,15 +1172,15 @@ build_info: engine: "docker|native" output: "target/nextflow/salmon/salmon_quant" executable: "target/nextflow/salmon/salmon_quant/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/salmon/salmon_quant/main.nf b/target/nextflow/salmon/salmon_quant/main.nf index 804e441f..b32f81a7 100644 --- a/target/nextflow/salmon/salmon_quant/main.nf +++ b/target/nextflow/salmon/salmon_quant/main.nf @@ -1,6 +1,6 @@ // salmon_quant main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -763,8 +763,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -786,7 +789,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -824,8 +840,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1601,8 +1620,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2629,30 +2648,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2723,7 +2743,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -3937,15 +3957,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/salmon/salmon_quant", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/salmon/salmon_quant/nextflow_schema.json b/target/nextflow/salmon/salmon_quant/nextflow_schema.json index 80b21398..ac2d9787 100644 --- a/target/nextflow/salmon/salmon_quant/nextflow_schema.json +++ b/target/nextflow/salmon/salmon_quant/nextflow_schema.json @@ -50,8 +50,8 @@ "unmated_reads": { "type": "string", - "description": "Type: List of `file`, example: `sample.fq.gz`, multiple_sep: `\":\"`. List of files containing unmated reads of (e", - "help_text": "Type: List of `file`, example: `sample.fq.gz`, multiple_sep: `\":\"`. List of files containing unmated reads of (e.g. single-end reads).\n" + "description": "Type: List of `file`, example: `sample.fq.gz`, multiple_sep: `\";\"`. List of files containing unmated reads of (e", + "help_text": "Type: List of `file`, example: `sample.fq.gz`, multiple_sep: `\";\"`. List of files containing unmated reads of (e.g. single-end reads).\n" } @@ -60,8 +60,8 @@ "mates1": { "type": "string", - "description": "Type: List of `file`, example: `sample_1.fq.gz`, multiple_sep: `\":\"`. File containing the #1 mates", - "help_text": "Type: List of `file`, example: `sample_1.fq.gz`, multiple_sep: `\":\"`. File containing the #1 mates.\n" + "description": "Type: List of `file`, example: `sample_1.fq.gz`, multiple_sep: `\";\"`. File containing the #1 mates", + "help_text": "Type: List of `file`, example: `sample_1.fq.gz`, multiple_sep: `\";\"`. File containing the #1 mates.\n" } @@ -70,8 +70,8 @@ "mates2": { "type": "string", - "description": "Type: List of `file`, example: `sample_2.fq.gz`, multiple_sep: `\":\"`. File containing the #2 mates", - "help_text": "Type: List of `file`, example: `sample_2.fq.gz`, multiple_sep: `\":\"`. File containing the #2 mates.\n" + "description": "Type: List of `file`, example: `sample_2.fq.gz`, multiple_sep: `\";\"`. File containing the #2 mates", + "help_text": "Type: List of `file`, example: `sample_2.fq.gz`, multiple_sep: `\";\"`. File containing the #2 mates.\n" } @@ -101,8 +101,8 @@ "alignments": { "type": "string", - "description": "Type: List of `file`, example: `sample.fq.gz`, multiple_sep: `\":\"`. Input alignment (BAM) file(s)", - "help_text": "Type: List of `file`, example: `sample.fq.gz`, multiple_sep: `\":\"`. Input alignment (BAM) file(s).\n" + "description": "Type: List of `file`, example: `sample.fq.gz`, multiple_sep: `\";\"`. Input alignment (BAM) file(s)", + "help_text": "Type: List of `file`, example: `sample.fq.gz`, multiple_sep: `\";\"`. Input alignment (BAM) file(s).\n" } @@ -217,8 +217,8 @@ "incompat_prior": { "type": "number", - "description": "Type: `double`, example: `0`. Set the prior probability that an alignment that disagrees with the specified library type (--lib_type) results from the true fragment origin", - "help_text": "Type: `double`, example: `0`. Set the prior probability that an alignment that disagrees with the specified library type (--lib_type) results from the true fragment origin. Setting this to 0 specifies that alignments that disagree with the library type should be \"impossible\", while setting it to 1 says that alignments that disagree with the library type are no less likely than those that do.\n" + "description": "Type: `double`, example: `0.0`. Set the prior probability that an alignment that disagrees with the specified library type (--lib_type) results from the true fragment origin", + "help_text": "Type: `double`, example: `0.0`. Set the prior probability that an alignment that disagrees with the specified library type (--lib_type) results from the true fragment origin. Setting this to 0 specifies that alignments that disagree with the library type should be \"impossible\", while setting it to 1 says that alignments that disagree with the library type are no less likely than those that do.\n" } @@ -258,8 +258,8 @@ "score_exp": { "type": "number", - "description": "Type: `double`, example: `1`. The factor by which sub-optimal alignment scores are downweighted to produce a probability", - "help_text": "Type: `double`, example: `1`. The factor by which sub-optimal alignment scores are downweighted to produce a probability. If the best alignment score for the current read is S, and the score for a particular alignment is w, then the probability will be computed porportional to exp( - scoreExp * (S-w) ).\n" + "description": "Type: `double`, example: `1.0`. The factor by which sub-optimal alignment scores are downweighted to produce a probability", + "help_text": "Type: `double`, example: `1.0`. The factor by which sub-optimal alignment scores are downweighted to produce a probability. If the best alignment score for the current read is S, and the score for a particular alignment is w, then the probability will be computed porportional to exp( - scoreExp * (S-w) ).\n" } @@ -360,8 +360,8 @@ "decoy_threshold": { "type": "number", - "description": "Type: `double`, example: `1`. [selective-alignment mode only]\nFor an alignemnt to an annotated transcript to be considered invalid, it must have an alignment score \u003c (decoy_threshold * bestDecoyScore)", - "help_text": "Type: `double`, example: `1`. [selective-alignment mode only]\nFor an alignemnt to an annotated transcript to be considered invalid, it must have an alignment score \u003c (decoy_threshold * bestDecoyScore). A value of 1.0 means that any alignment strictly worse than the best decoy alignment will be discarded. A smaller value will allow reads to be allocated to transcripts even if they strictly align better to the decoy sequence.\n" + "description": "Type: `double`, example: `1.0`. [selective-alignment mode only]\nFor an alignemnt to an annotated transcript to be considered invalid, it must have an alignment score \u003c (decoy_threshold * bestDecoyScore)", + "help_text": "Type: `double`, example: `1.0`. [selective-alignment mode only]\nFor an alignemnt to an annotated transcript to be considered invalid, it must have an alignment score \u003c (decoy_threshold * bestDecoyScore). A value of 1.0 means that any alignment strictly worse than the best decoy alignment will be discarded. A smaller value will allow reads to be allocated to transcripts even if they strictly align better to the decoy sequence.\n" } diff --git a/target/nextflow/samtools/samtools_collate/.config.vsh.yaml b/target/nextflow/samtools/samtools_collate/.config.vsh.yaml index d1b99575..799ea98e 100644 --- a/target/nextflow/samtools/samtools_collate/.config.vsh.yaml +++ b/target/nextflow/samtools/samtools_collate/.config.vsh.yaml @@ -263,15 +263,15 @@ build_info: engine: "docker|native" output: "target/nextflow/samtools/samtools_collate" executable: "target/nextflow/samtools/samtools_collate/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/samtools/samtools_collate/main.nf b/target/nextflow/samtools/samtools_collate/main.nf index f44604c5..f5177c50 100644 --- a/target/nextflow/samtools/samtools_collate/main.nf +++ b/target/nextflow/samtools/samtools_collate/main.nf @@ -1,6 +1,6 @@ // samtools_collate main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -763,8 +763,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -786,7 +789,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -824,8 +840,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1601,8 +1620,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2629,30 +2648,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2723,7 +2743,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -3113,15 +3133,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/samtools/samtools_collate", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/samtools/samtools_faidx/.config.vsh.yaml b/target/nextflow/samtools/samtools_faidx/.config.vsh.yaml index fbf79dc7..da27b1c2 100644 --- a/target/nextflow/samtools/samtools_faidx/.config.vsh.yaml +++ b/target/nextflow/samtools/samtools_faidx/.config.vsh.yaml @@ -242,15 +242,15 @@ build_info: engine: "docker|native" output: "target/nextflow/samtools/samtools_faidx" executable: "target/nextflow/samtools/samtools_faidx/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/samtools/samtools_faidx/main.nf b/target/nextflow/samtools/samtools_faidx/main.nf index ee0259f6..fa7a23a8 100644 --- a/target/nextflow/samtools/samtools_faidx/main.nf +++ b/target/nextflow/samtools/samtools_faidx/main.nf @@ -1,6 +1,6 @@ // samtools_faidx main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -763,8 +763,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -786,7 +789,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -824,8 +840,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1601,8 +1620,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2629,30 +2648,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2723,7 +2743,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -3085,15 +3105,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/samtools/samtools_faidx", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/samtools/samtools_fasta/.config.vsh.yaml b/target/nextflow/samtools/samtools_fasta/.config.vsh.yaml index a3e5b79b..92702a42 100644 --- a/target/nextflow/samtools/samtools_fasta/.config.vsh.yaml +++ b/target/nextflow/samtools/samtools_fasta/.config.vsh.yaml @@ -432,15 +432,15 @@ build_info: engine: "docker|native" output: "target/nextflow/samtools/samtools_fasta" executable: "target/nextflow/samtools/samtools_fasta/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/samtools/samtools_fasta/main.nf b/target/nextflow/samtools/samtools_fasta/main.nf index e23b0258..373250f3 100644 --- a/target/nextflow/samtools/samtools_fasta/main.nf +++ b/target/nextflow/samtools/samtools_fasta/main.nf @@ -1,6 +1,6 @@ // samtools_fasta main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -763,8 +763,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -786,7 +789,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -824,8 +840,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1601,8 +1620,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2629,30 +2648,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2723,7 +2743,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -3277,15 +3297,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/samtools/samtools_fasta", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/samtools/samtools_fastq/.config.vsh.yaml b/target/nextflow/samtools/samtools_fastq/.config.vsh.yaml index 4d1aa8fe..8776e269 100644 --- a/target/nextflow/samtools/samtools_fastq/.config.vsh.yaml +++ b/target/nextflow/samtools/samtools_fastq/.config.vsh.yaml @@ -432,15 +432,15 @@ build_info: engine: "docker|native" output: "target/nextflow/samtools/samtools_fastq" executable: "target/nextflow/samtools/samtools_fastq/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/samtools/samtools_fastq/main.nf b/target/nextflow/samtools/samtools_fastq/main.nf index 77d4f129..cd541c63 100644 --- a/target/nextflow/samtools/samtools_fastq/main.nf +++ b/target/nextflow/samtools/samtools_fastq/main.nf @@ -1,6 +1,6 @@ // samtools_fastq main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -763,8 +763,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -786,7 +789,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -824,8 +840,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1601,8 +1620,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2629,30 +2648,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2723,7 +2743,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -3277,15 +3297,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/samtools/samtools_fastq", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/samtools/samtools_flagstat/.config.vsh.yaml b/target/nextflow/samtools/samtools_flagstat/.config.vsh.yaml index f0c69b30..52863308 100644 --- a/target/nextflow/samtools/samtools_flagstat/.config.vsh.yaml +++ b/target/nextflow/samtools/samtools_flagstat/.config.vsh.yaml @@ -172,15 +172,15 @@ build_info: engine: "docker|native" output: "target/nextflow/samtools/samtools_flagstat" executable: "target/nextflow/samtools/samtools_flagstat/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/samtools/samtools_flagstat/main.nf b/target/nextflow/samtools/samtools_flagstat/main.nf index 683f8c50..31f2be32 100644 --- a/target/nextflow/samtools/samtools_flagstat/main.nf +++ b/target/nextflow/samtools/samtools_flagstat/main.nf @@ -1,6 +1,6 @@ // samtools_flagstat main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -763,8 +763,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -786,7 +789,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -824,8 +840,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1601,8 +1620,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2629,30 +2648,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2723,7 +2743,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -3001,15 +3021,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/samtools/samtools_flagstat", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/samtools/samtools_idxstats/.config.vsh.yaml b/target/nextflow/samtools/samtools_idxstats/.config.vsh.yaml index c7c917ef..e42bfd04 100644 --- a/target/nextflow/samtools/samtools_idxstats/.config.vsh.yaml +++ b/target/nextflow/samtools/samtools_idxstats/.config.vsh.yaml @@ -182,15 +182,15 @@ build_info: engine: "docker|native" output: "target/nextflow/samtools/samtools_idxstats" executable: "target/nextflow/samtools/samtools_idxstats/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/samtools/samtools_idxstats/main.nf b/target/nextflow/samtools/samtools_idxstats/main.nf index 7bfe9ea1..b2f18531 100644 --- a/target/nextflow/samtools/samtools_idxstats/main.nf +++ b/target/nextflow/samtools/samtools_idxstats/main.nf @@ -1,6 +1,6 @@ // samtools_idxstats main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -763,8 +763,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -786,7 +789,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -824,8 +840,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1601,8 +1620,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2629,30 +2648,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2723,7 +2743,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -3013,15 +3033,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/samtools/samtools_idxstats", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/samtools/samtools_index/.config.vsh.yaml b/target/nextflow/samtools/samtools_index/.config.vsh.yaml index 7b3f46a5..96f2c0f9 100644 --- a/target/nextflow/samtools/samtools_index/.config.vsh.yaml +++ b/target/nextflow/samtools/samtools_index/.config.vsh.yaml @@ -188,15 +188,15 @@ build_info: engine: "docker|native" output: "target/nextflow/samtools/samtools_index" executable: "target/nextflow/samtools/samtools_index/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/samtools/samtools_index/main.nf b/target/nextflow/samtools/samtools_index/main.nf index 38e42575..8faf0737 100644 --- a/target/nextflow/samtools/samtools_index/main.nf +++ b/target/nextflow/samtools/samtools_index/main.nf @@ -1,6 +1,6 @@ // samtools_index main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -763,8 +763,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -786,7 +789,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -824,8 +840,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1601,8 +1620,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2629,30 +2648,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2723,7 +2743,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -3026,15 +3046,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/samtools/samtools_index", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/samtools/samtools_sort/.config.vsh.yaml b/target/nextflow/samtools/samtools_sort/.config.vsh.yaml index 0db5d84f..75bcb45e 100644 --- a/target/nextflow/samtools/samtools_sort/.config.vsh.yaml +++ b/target/nextflow/samtools/samtools_sort/.config.vsh.yaml @@ -331,15 +331,15 @@ build_info: engine: "docker|native" output: "target/nextflow/samtools/samtools_sort" executable: "target/nextflow/samtools/samtools_sort/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/samtools/samtools_sort/main.nf b/target/nextflow/samtools/samtools_sort/main.nf index 21332804..1b78408f 100644 --- a/target/nextflow/samtools/samtools_sort/main.nf +++ b/target/nextflow/samtools/samtools_sort/main.nf @@ -1,6 +1,6 @@ // samtools_sort main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -763,8 +763,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -786,7 +789,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -824,8 +840,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1601,8 +1620,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2629,30 +2648,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2723,7 +2743,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -3198,15 +3218,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/samtools/samtools_sort", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/samtools/samtools_stats/.config.vsh.yaml b/target/nextflow/samtools/samtools_stats/.config.vsh.yaml index a1dd9ffc..8e06b8aa 100644 --- a/target/nextflow/samtools/samtools_stats/.config.vsh.yaml +++ b/target/nextflow/samtools/samtools_stats/.config.vsh.yaml @@ -400,15 +400,15 @@ build_info: engine: "docker|native" output: "target/nextflow/samtools/samtools_stats" executable: "target/nextflow/samtools/samtools_stats/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/samtools/samtools_stats/main.nf b/target/nextflow/samtools/samtools_stats/main.nf index 23b8bfe5..01cd1257 100644 --- a/target/nextflow/samtools/samtools_stats/main.nf +++ b/target/nextflow/samtools/samtools_stats/main.nf @@ -1,6 +1,6 @@ // samtools_stats main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -763,8 +763,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -786,7 +789,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -824,8 +840,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1601,8 +1620,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2629,30 +2648,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2723,7 +2743,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -3268,15 +3288,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/samtools/samtools_stats", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/samtools/samtools_stats/nextflow_schema.json b/target/nextflow/samtools/samtools_stats/nextflow_schema.json index 38f8b084..147c8c22 100644 --- a/target/nextflow/samtools/samtools_stats/nextflow_schema.json +++ b/target/nextflow/samtools/samtools_stats/nextflow_schema.json @@ -47,8 +47,8 @@ "coverage": { "type": "string", - "description": "Type: List of `integer`, example: `1:1000:1`, multiple_sep: `\":\"`. Coverage distribution min;max;step", - "help_text": "Type: List of `integer`, example: `1:1000:1`, multiple_sep: `\":\"`. Coverage distribution min;max;step. Default: [1, 1000, 1].\n" + "description": "Type: List of `integer`, example: `1;1000;1`, multiple_sep: `\";\"`. Coverage distribution min;max;step", + "help_text": "Type: List of `integer`, example: `1;1000;1`, multiple_sep: `\";\"`. Coverage distribution min;max;step. Default: [1, 1000, 1].\n" } diff --git a/target/nextflow/samtools/samtools_view/.config.vsh.yaml b/target/nextflow/samtools/samtools_view/.config.vsh.yaml index 265cd362..a5e1cfc9 100644 --- a/target/nextflow/samtools/samtools_view/.config.vsh.yaml +++ b/target/nextflow/samtools/samtools_view/.config.vsh.yaml @@ -664,15 +664,15 @@ build_info: engine: "docker|native" output: "target/nextflow/samtools/samtools_view" executable: "target/nextflow/samtools/samtools_view/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/samtools/samtools_view/main.nf b/target/nextflow/samtools/samtools_view/main.nf index e2fd8c6e..22f2f7ff 100644 --- a/target/nextflow/samtools/samtools_view/main.nf +++ b/target/nextflow/samtools/samtools_view/main.nf @@ -1,6 +1,6 @@ // samtools_view main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -763,8 +763,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -786,7 +789,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -824,8 +840,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1601,8 +1620,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2629,30 +2648,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2723,7 +2743,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -3449,15 +3469,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/samtools/samtools_view", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/seqtk/seqtk_sample/.config.vsh.yaml b/target/nextflow/seqtk/seqtk_sample/.config.vsh.yaml index 5377b3c3..e5589f1f 100644 --- a/target/nextflow/seqtk/seqtk_sample/.config.vsh.yaml +++ b/target/nextflow/seqtk/seqtk_sample/.config.vsh.yaml @@ -172,15 +172,15 @@ build_info: engine: "docker|native" output: "target/nextflow/seqtk/seqtk_sample" executable: "target/nextflow/seqtk/seqtk_sample/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/seqtk/seqtk_sample/main.nf b/target/nextflow/seqtk/seqtk_sample/main.nf index c2b91505..db067f46 100644 --- a/target/nextflow/seqtk/seqtk_sample/main.nf +++ b/target/nextflow/seqtk/seqtk_sample/main.nf @@ -1,6 +1,6 @@ // seqtk_sample main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -763,8 +763,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -786,7 +789,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -824,8 +840,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1601,8 +1620,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2629,30 +2648,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2723,7 +2743,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -3003,15 +3023,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/seqtk/seqtk_sample", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/seqtk/seqtk_subseq/.config.vsh.yaml b/target/nextflow/seqtk/seqtk_subseq/.config.vsh.yaml index e6f8dd4b..09b70c32 100644 --- a/target/nextflow/seqtk/seqtk_subseq/.config.vsh.yaml +++ b/target/nextflow/seqtk/seqtk_subseq/.config.vsh.yaml @@ -195,15 +195,15 @@ build_info: engine: "docker|native" output: "target/nextflow/seqtk/seqtk_subseq" executable: "target/nextflow/seqtk/seqtk_subseq/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/seqtk/seqtk_subseq/main.nf b/target/nextflow/seqtk/seqtk_subseq/main.nf index 8d2a6a62..ad08a928 100644 --- a/target/nextflow/seqtk/seqtk_subseq/main.nf +++ b/target/nextflow/seqtk/seqtk_subseq/main.nf @@ -1,6 +1,6 @@ // seqtk_subseq main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -763,8 +763,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -786,7 +789,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -824,8 +840,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1601,8 +1620,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2629,30 +2648,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2723,7 +2743,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -3033,15 +3053,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/seqtk/seqtk_subseq", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/star/star_align_reads/.config.vsh.yaml b/target/nextflow/star/star_align_reads/.config.vsh.yaml index baafcdb4..955e2848 100644 --- a/target/nextflow/star/star_align_reads/.config.vsh.yaml +++ b/target/nextflow/star/star_align_reads/.config.vsh.yaml @@ -2662,15 +2662,15 @@ build_info: engine: "docker|native" output: "target/nextflow/star/star_align_reads" executable: "target/nextflow/star/star_align_reads/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/star/star_align_reads/main.nf b/target/nextflow/star/star_align_reads/main.nf index e3d4fd07..84494fda 100644 --- a/target/nextflow/star/star_align_reads/main.nf +++ b/target/nextflow/star/star_align_reads/main.nf @@ -1,6 +1,6 @@ // star_align_reads main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -764,8 +764,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -787,7 +790,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -825,8 +841,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1602,8 +1621,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2630,30 +2649,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2724,7 +2744,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -5916,15 +5936,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/star/star_align_reads", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/star/star_align_reads/nextflow_schema.json b/target/nextflow/star/star_align_reads/nextflow_schema.json index 82127551..7afc49c8 100644 --- a/target/nextflow/star/star_align_reads/nextflow_schema.json +++ b/target/nextflow/star/star_align_reads/nextflow_schema.json @@ -17,8 +17,8 @@ "input": { "type": "string", - "description": "Type: List of `file`, required, example: `mysample_S1_L001_R1_001.fastq.gz`, multiple_sep: `\":\"`. The single-end or paired-end R1 FastQ files to be processed", - "help_text": "Type: List of `file`, required, example: `mysample_S1_L001_R1_001.fastq.gz`, multiple_sep: `\":\"`. The single-end or paired-end R1 FastQ files to be processed." + "description": "Type: List of `file`, required, example: `mysample_S1_L001_R1_001.fastq.gz`, multiple_sep: `\";\"`. The single-end or paired-end R1 FastQ files to be processed", + "help_text": "Type: List of `file`, required, example: `mysample_S1_L001_R1_001.fastq.gz`, multiple_sep: `\";\"`. The single-end or paired-end R1 FastQ files to be processed." } @@ -27,8 +27,8 @@ "input_r2": { "type": "string", - "description": "Type: List of `file`, example: `mysample_S1_L001_R2_001.fastq.gz`, multiple_sep: `\":\"`. The paired-end R2 FastQ files to be processed", - "help_text": "Type: List of `file`, example: `mysample_S1_L001_R2_001.fastq.gz`, multiple_sep: `\":\"`. The paired-end R2 FastQ files to be processed. Only required if --input is a paired-end R1 file." + "description": "Type: List of `file`, example: `mysample_S1_L001_R2_001.fastq.gz`, multiple_sep: `\";\"`. The paired-end R2 FastQ files to be processed", + "help_text": "Type: List of `file`, example: `mysample_S1_L001_R2_001.fastq.gz`, multiple_sep: `\";\"`. The paired-end R2 FastQ files to be processed. Only required if --input is a paired-end R1 file." } @@ -131,6 +131,2046 @@ } +} +}, + + + "run parameters" : { + "title": "Run Parameters", + "type": "object", + "description": "No description", + "properties": { + + + "run_rng_seed": { + "type": + "integer", + "description": "Type: `integer`, example: `777`. random number generator seed", + "help_text": "Type: `integer`, example: `777`. random number generator seed." + + } + + +} +}, + + + "genome parameters" : { + "title": "Genome Parameters", + "type": "object", + "description": "No description", + "properties": { + + + "genome_dir": { + "type": + "string", + "description": "Type: `file`, required, example: `./GenomeDir`. path to the directory where genome files are stored (for --runMode alignReads) or will be generated (for --runMode generateGenome)", + "help_text": "Type: `file`, required, example: `./GenomeDir`. path to the directory where genome files are stored (for --runMode alignReads) or will be generated (for --runMode generateGenome)" + + } + + + , + "genome_load": { + "type": + "string", + "description": "Type: `string`, example: `NoSharedMemory`. mode of shared memory usage for the genome files", + "help_text": "Type: `string`, example: `NoSharedMemory`. mode of shared memory usage for the genome files. Only used with --runMode alignReads.\n\n- LoadAndKeep ... load genome into shared and keep it in memory after run\n- LoadAndRemove ... load genome into shared but remove it after run\n- LoadAndExit ... load genome into shared memory and exit, keeping the genome in memory for future runs\n- Remove ... do not map anything, just remove loaded genome from memory\n- NoSharedMemory ... do not use shared memory, each job will have its own private copy of the genome" + + } + + + , + "genome_fasta_files": { + "type": + "string", + "description": "Type: List of `file`, multiple_sep: `\";\"`. path(s) to the fasta files with the genome sequences, separated by spaces", + "help_text": "Type: List of `file`, multiple_sep: `\";\"`. path(s) to the fasta files with the genome sequences, separated by spaces. These files should be plain text FASTA files, they *cannot* be zipped.\n\nRequired for the genome generation (--runMode genomeGenerate). Can also be used in the mapping (--runMode alignReads) to add extra (new) sequences to the genome (e.g. spike-ins)." + + } + + + , + "genome_file_sizes": { + "type": + "string", + "description": "Type: List of `integer`, example: `0`, multiple_sep: `\";\"`. genome files exact sizes in bytes", + "help_text": "Type: List of `integer`, example: `0`, multiple_sep: `\";\"`. genome files exact sizes in bytes. Typically, this should not be defined by the user." + + } + + + , + "genome_transform_output": { + "type": + "string", + "description": "Type: List of `string`, multiple_sep: `\";\"`. which output to transform back to original genome\n\n- SAM ", + "help_text": "Type: List of `string`, multiple_sep: `\";\"`. which output to transform back to original genome\n\n- SAM ... SAM/BAM alignments\n- SJ ... splice junctions (SJ.out.tab)\n- Quant ... quantifications (from --quant_mode option)\n- None ... no transformation of the output" + + } + + + , + "genome_chr_set_mitochondrial": { + "type": + "string", + "description": "Type: List of `string`, example: `chrM;M;MT`, multiple_sep: `\";\"`. names of the mitochondrial chromosomes", + "help_text": "Type: List of `string`, example: `chrM;M;MT`, multiple_sep: `\";\"`. names of the mitochondrial chromosomes. Presently only used for STARsolo statistics output/" + + } + + +} +}, + + + "splice junctions database" : { + "title": "Splice Junctions Database", + "type": "object", + "description": "No description", + "properties": { + + + "sjdb_file_chr_start_end": { + "type": + "string", + "description": "Type: List of `string`, multiple_sep: `\";\"`. path to the files with genomic coordinates (chr \u003ctab\u003e start \u003ctab\u003e end \u003ctab\u003e strand) for the splice junction introns", + "help_text": "Type: List of `string`, multiple_sep: `\";\"`. path to the files with genomic coordinates (chr \u003ctab\u003e start \u003ctab\u003e end \u003ctab\u003e strand) for the splice junction introns. Multiple files can be supplied and will be concatenated." + + } + + + , + "sjdb_gtf_file": { + "type": + "string", + "description": "Type: `file`. path to the GTF file with annotations", + "help_text": "Type: `file`. path to the GTF file with annotations" + + } + + + , + "sjdb_gtf_chr_prefix": { + "type": + "string", + "description": "Type: `string`. prefix for chromosome names in a GTF file (e", + "help_text": "Type: `string`. prefix for chromosome names in a GTF file (e.g. \u0027chr\u0027 for using ENSMEBL annotations with UCSC genomes)" + + } + + + , + "sjdb_gtf_feature_exon": { + "type": + "string", + "description": "Type: `string`, example: `exon`. feature type in GTF file to be used as exons for building transcripts", + "help_text": "Type: `string`, example: `exon`. feature type in GTF file to be used as exons for building transcripts" + + } + + + , + "sjdb_gtf_tag_exon_parent_transcript": { + "type": + "string", + "description": "Type: `string`, example: `transcript_id`. GTF attribute name for parent transcript ID (default \"transcript_id\" works for GTF files)", + "help_text": "Type: `string`, example: `transcript_id`. GTF attribute name for parent transcript ID (default \"transcript_id\" works for GTF files)" + + } + + + , + "sjdb_gtf_tag_exon_parent_gene": { + "type": + "string", + "description": "Type: `string`, example: `gene_id`. GTF attribute name for parent gene ID (default \"gene_id\" works for GTF files)", + "help_text": "Type: `string`, example: `gene_id`. GTF attribute name for parent gene ID (default \"gene_id\" works for GTF files)" + + } + + + , + "sjdb_gtf_tag_exon_parent_gene_name": { + "type": + "string", + "description": "Type: List of `string`, example: `gene_name`, multiple_sep: `\";\"`. GTF attribute name for parent gene name", + "help_text": "Type: List of `string`, example: `gene_name`, multiple_sep: `\";\"`. GTF attribute name for parent gene name" + + } + + + , + "sjdb_gtf_tag_exon_parent_gene_type": { + "type": + "string", + "description": "Type: List of `string`, example: `gene_type;gene_biotype`, multiple_sep: `\";\"`. GTF attribute name for parent gene type", + "help_text": "Type: List of `string`, example: `gene_type;gene_biotype`, multiple_sep: `\";\"`. GTF attribute name for parent gene type" + + } + + + , + "sjdb_overhang": { + "type": + "integer", + "description": "Type: `integer`, example: `100`. length of the donor/acceptor sequence on each side of the junctions, ideally = (mate_length - 1)", + "help_text": "Type: `integer`, example: `100`. length of the donor/acceptor sequence on each side of the junctions, ideally = (mate_length - 1)" + + } + + + , + "sjdb_score": { + "type": + "integer", + "description": "Type: `integer`, example: `2`. extra alignment score for alignments that cross database junctions", + "help_text": "Type: `integer`, example: `2`. extra alignment score for alignments that cross database junctions" + + } + + + , + "sjdb_insert_save": { + "type": + "string", + "description": "Type: `string`, example: `Basic`. which files to save when sjdb junctions are inserted on the fly at the mapping step\n\n- Basic ", + "help_text": "Type: `string`, example: `Basic`. which files to save when sjdb junctions are inserted on the fly at the mapping step\n\n- Basic ... only small junction / transcript files\n- All ... all files including big Genome, SA and SAindex - this will create a complete genome directory" + + } + + +} +}, + + + "variation parameters" : { + "title": "Variation parameters", + "type": "object", + "description": "No description", + "properties": { + + + "var_vcf_file": { + "type": + "string", + "description": "Type: `string`. path to the VCF file that contains variation data", + "help_text": "Type: `string`. path to the VCF file that contains variation data. The 10th column should contain the genotype information, e.g. 0/1" + + } + + +} +}, + + + "read parameters" : { + "title": "Read Parameters", + "type": "object", + "description": "No description", + "properties": { + + + "read_files_type": { + "type": + "string", + "description": "Type: `string`, example: `Fastx`. format of input read files\n\n- Fastx ", + "help_text": "Type: `string`, example: `Fastx`. format of input read files\n\n- Fastx ... FASTA or FASTQ\n- SAM SE ... SAM or BAM single-end reads; for BAM use --read_files_command samtools view\n- SAM PE ... SAM or BAM paired-end reads; for BAM use --read_files_command samtools view" + + } + + + , + "read_files_sam_attr_keep": { + "type": + "string", + "description": "Type: List of `string`, example: `All`, multiple_sep: `\";\"`. for --read_files_type SAM SE/PE, which SAM tags to keep in the output BAM, e", + "help_text": "Type: List of `string`, example: `All`, multiple_sep: `\";\"`. for --read_files_type SAM SE/PE, which SAM tags to keep in the output BAM, e.g.: --readFilesSAMtagsKeep RG PL\n\n- All ... keep all tags\n- None ... do not keep any tags" + + } + + + , + "read_files_manifest": { + "type": + "string", + "description": "Type: `file`. path to the \"manifest\" file with the names of read files", + "help_text": "Type: `file`. path to the \"manifest\" file with the names of read files. The manifest file should contain 3 tab-separated columns:\n\npaired-end reads: read1_file_name $tab$ read2_file_name $tab$ read_group_line.\nsingle-end reads: read1_file_name $tab$ - $tab$ read_group_line.\nSpaces, but not tabs are allowed in file names.\nIf read_group_line does not start with ID:, it can only contain one ID field, and ID: will be added to it.\nIf read_group_line starts with ID:, it can contain several fields separated by $tab$, and all fields will be be copied verbatim into SAM @RG header line." + + } + + + , + "read_files_prefix": { + "type": + "string", + "description": "Type: `string`. prefix for the read files names, i", + "help_text": "Type: `string`. prefix for the read files names, i.e. it will be added in front of the strings in --readFilesIn" + + } + + + , + "read_files_command": { + "type": + "string", + "description": "Type: List of `string`, multiple_sep: `\";\"`. command line to execute for each of the input file", + "help_text": "Type: List of `string`, multiple_sep: `\";\"`. command line to execute for each of the input file. This command should generate FASTA or FASTQ text and send it to stdout\n\nFor example: zcat - to uncompress .gz files, bzcat - to uncompress .bz2 files, etc." + + } + + + , + "read_map_number": { + "type": + "integer", + "description": "Type: `integer`, example: `-1`. number of reads to map from the beginning of the file\n\n-1: map all reads", + "help_text": "Type: `integer`, example: `-1`. number of reads to map from the beginning of the file\n\n-1: map all reads" + + } + + + , + "read_mates_lengths_in": { + "type": + "string", + "description": "Type: `string`, example: `NotEqual`. Equal/NotEqual - lengths of names,sequences,qualities for both mates are the same / not the same", + "help_text": "Type: `string`, example: `NotEqual`. Equal/NotEqual - lengths of names,sequences,qualities for both mates are the same / not the same. NotEqual is safe in all situations." + + } + + + , + "read_name_separator": { + "type": + "string", + "description": "Type: List of `string`, example: `/`, multiple_sep: `\";\"`. character(s) separating the part of the read names that will be trimmed in output (read name after space is always trimmed)", + "help_text": "Type: List of `string`, example: `/`, multiple_sep: `\";\"`. character(s) separating the part of the read names that will be trimmed in output (read name after space is always trimmed)" + + } + + + , + "read_quality_score_base": { + "type": + "integer", + "description": "Type: `integer`, example: `33`. number to be subtracted from the ASCII code to get Phred quality score", + "help_text": "Type: `integer`, example: `33`. number to be subtracted from the ASCII code to get Phred quality score" + + } + + +} +}, + + + "read clipping" : { + "title": "Read Clipping", + "type": "object", + "description": "No description", + "properties": { + + + "clip_adapter_type": { + "type": + "string", + "description": "Type: `string`, example: `Hamming`. adapter clipping type\n\n- Hamming ", + "help_text": "Type: `string`, example: `Hamming`. adapter clipping type\n\n- Hamming ... adapter clipping based on Hamming distance, with the number of mismatches controlled by --clip5pAdapterMMp\n- CellRanger4 ... 5p and 3p adapter clipping similar to CellRanger4. Utilizes Opal package by Martin Sosic: https://github.com/Martinsos/opal\n- None ... no adapter clipping, all other clip* parameters are disregarded" + + } + + + , + "clip3p_nbases": { + "type": + "string", + "description": "Type: List of `integer`, example: `0`, multiple_sep: `\";\"`. number(s) of bases to clip from 3p of each mate", + "help_text": "Type: List of `integer`, example: `0`, multiple_sep: `\";\"`. number(s) of bases to clip from 3p of each mate. If one value is given, it will be assumed the same for both mates." + + } + + + , + "clip3p_adapter_seq": { + "type": + "string", + "description": "Type: List of `string`, multiple_sep: `\";\"`. adapter sequences to clip from 3p of each mate", + "help_text": "Type: List of `string`, multiple_sep: `\";\"`. adapter sequences to clip from 3p of each mate. If one value is given, it will be assumed the same for both mates.\n\n- polyA ... polyA sequence with the length equal to read length" + + } + + + , + "clip3p_adapter_mm_p": { + "type": + "string", + "description": "Type: List of `double`, example: `0.1`, multiple_sep: `\";\"`. max proportion of mismatches for 3p adapter clipping for each mate", + "help_text": "Type: List of `double`, example: `0.1`, multiple_sep: `\";\"`. max proportion of mismatches for 3p adapter clipping for each mate. If one value is given, it will be assumed the same for both mates." + + } + + + , + "clip3p_after_adapter_nbases": { + "type": + "string", + "description": "Type: List of `integer`, example: `0`, multiple_sep: `\";\"`. number of bases to clip from 3p of each mate after the adapter clipping", + "help_text": "Type: List of `integer`, example: `0`, multiple_sep: `\";\"`. number of bases to clip from 3p of each mate after the adapter clipping. If one value is given, it will be assumed the same for both mates." + + } + + + , + "clip5p_nbases": { + "type": + "string", + "description": "Type: List of `integer`, example: `0`, multiple_sep: `\";\"`. number(s) of bases to clip from 5p of each mate", + "help_text": "Type: List of `integer`, example: `0`, multiple_sep: `\";\"`. number(s) of bases to clip from 5p of each mate. If one value is given, it will be assumed the same for both mates." + + } + + +} +}, + + + "limits" : { + "title": "Limits", + "type": "object", + "description": "No description", + "properties": { + + + "limit_genome_generate_ram": { + "type": + "string", + "description": "Type: `long`, example: `31000000000`. maximum available RAM (bytes) for genome generation", + "help_text": "Type: `long`, example: `31000000000`. maximum available RAM (bytes) for genome generation" + + } + + + , + "limit_io_buffer_size": { + "type": + "string", + "description": "Type: List of `long`, example: `30000000;50000000`, multiple_sep: `\";\"`. max available buffers size (bytes) for input/output, per thread", + "help_text": "Type: List of `long`, example: `30000000;50000000`, multiple_sep: `\";\"`. max available buffers size (bytes) for input/output, per thread" + + } + + + , + "limit_out_sam_one_read_bytes": { + "type": + "string", + "description": "Type: `long`, example: `100000`. max size of the SAM record (bytes) for one read", + "help_text": "Type: `long`, example: `100000`. max size of the SAM record (bytes) for one read. Recommended value: \u003e(2*(LengthMate1+LengthMate2+100)*outFilterMultimapNmax" + + } + + + , + "limit_out_sj_one_read": { + "type": + "integer", + "description": "Type: `integer`, example: `1000`. max number of junctions for one read (including all multi-mappers)", + "help_text": "Type: `integer`, example: `1000`. max number of junctions for one read (including all multi-mappers)" + + } + + + , + "limit_out_sj_collapsed": { + "type": + "integer", + "description": "Type: `integer`, example: `1000000`. max number of collapsed junctions", + "help_text": "Type: `integer`, example: `1000000`. max number of collapsed junctions" + + } + + + , + "limit_bam_sort_ram": { + "type": + "string", + "description": "Type: `long`, example: `0`. maximum available RAM (bytes) for sorting BAM", + "help_text": "Type: `long`, example: `0`. maximum available RAM (bytes) for sorting BAM. If =0, it will be set to the genome index size. 0 value can only be used with --genome_load NoSharedMemory option." + + } + + + , + "limit_sjdb_insert_nsj": { + "type": + "integer", + "description": "Type: `integer`, example: `1000000`. maximum number of junctions to be inserted to the genome on the fly at the mapping stage, including those from annotations and those detected in the 1st step of the 2-pass run", + "help_text": "Type: `integer`, example: `1000000`. maximum number of junctions to be inserted to the genome on the fly at the mapping stage, including those from annotations and those detected in the 1st step of the 2-pass run" + + } + + + , + "limit_nreads_soft": { + "type": + "integer", + "description": "Type: `integer`, example: `-1`. soft limit on the number of reads", + "help_text": "Type: `integer`, example: `-1`. soft limit on the number of reads" + + } + + +} +}, + + + "output: general" : { + "title": "Output: general", + "type": "object", + "description": "No description", + "properties": { + + + "out_tmp_keep": { + "type": + "string", + "description": "Type: `string`. whether to keep the temporary files after STAR runs is finished\n\n- None ", + "help_text": "Type: `string`. whether to keep the temporary files after STAR runs is finished\n\n- None ... remove all temporary files\n- All ... keep all files" + + } + + + , + "out_std": { + "type": + "string", + "description": "Type: `string`, example: `Log`. which output will be directed to stdout (standard out)\n\n- Log ", + "help_text": "Type: `string`, example: `Log`. which output will be directed to stdout (standard out)\n\n- Log ... log messages\n- SAM ... alignments in SAM format (which normally are output to Aligned.out.sam file), normal standard output will go into Log.std.out\n- BAM_Unsorted ... alignments in BAM format, unsorted. Requires --out_sam_type BAM Unsorted\n- BAM_SortedByCoordinate ... alignments in BAM format, sorted by coordinate. Requires --out_sam_type BAM SortedByCoordinate\n- BAM_Quant ... alignments to transcriptome in BAM format, unsorted. Requires --quant_mode TranscriptomeSAM" + + } + + + , + "out_reads_unmapped": { + "type": + "string", + "description": "Type: `string`. output of unmapped and partially mapped (i", + "help_text": "Type: `string`. output of unmapped and partially mapped (i.e. mapped only one mate of a paired end read) reads in separate file(s).\n\n- None ... no output\n- Fastx ... output in separate fasta/fastq files, Unmapped.out.mate1/2" + + } + + + , + "out_qs_conversion_add": { + "type": + "integer", + "description": "Type: `integer`, example: `0`. add this number to the quality score (e", + "help_text": "Type: `integer`, example: `0`. add this number to the quality score (e.g. to convert from Illumina to Sanger, use -31)" + + } + + + , + "out_multimapper_order": { + "type": + "string", + "description": "Type: `string`, example: `Old_2.4`. order of multimapping alignments in the output files\n\n- Old_2", + "help_text": "Type: `string`, example: `Old_2.4`. order of multimapping alignments in the output files\n\n- Old_2.4 ... quasi-random order used before 2.5.0\n- Random ... random order of alignments for each multi-mapper. Read mates (pairs) are always adjacent, all alignment for each read stay together. This option will become default in the future releases." + + } + + +} +}, + + + "output: sam and bam" : { + "title": "Output: SAM and BAM", + "type": "object", + "description": "No description", + "properties": { + + + "out_sam_type": { + "type": + "string", + "description": "Type: List of `string`, example: `SAM`, multiple_sep: `\";\"`. type of SAM/BAM output\n\n1st word:\n- BAM ", + "help_text": "Type: List of `string`, example: `SAM`, multiple_sep: `\";\"`. type of SAM/BAM output\n\n1st word:\n- BAM ... output BAM without sorting\n- SAM ... output SAM without sorting\n- None ... no SAM/BAM output\n2nd, 3rd:\n- Unsorted ... standard unsorted\n- SortedByCoordinate ... sorted by coordinate. This option will allocate extra memory for sorting which can be specified by --limit_bam_sort_ram." + + } + + + , + "out_sam_mode": { + "type": + "string", + "description": "Type: `string`, example: `Full`. mode of SAM output\n\n- None ", + "help_text": "Type: `string`, example: `Full`. mode of SAM output\n\n- None ... no SAM output\n- Full ... full SAM output\n- NoQS ... full SAM but without quality scores" + + } + + + , + "out_sam_strand_field": { + "type": + "string", + "description": "Type: `string`. Cufflinks-like strand field flag\n\n- None ", + "help_text": "Type: `string`. Cufflinks-like strand field flag\n\n- None ... not used\n- intronMotif ... strand derived from the intron motif. This option changes the output alignments: reads with inconsistent and/or non-canonical introns are filtered out." + + } + + + , + "out_sam_attributes": { + "type": + "string", + "description": "Type: List of `string`, example: `Standard`, multiple_sep: `\";\"`. a string of desired SAM attributes, in the order desired for the output SAM", + "help_text": "Type: List of `string`, example: `Standard`, multiple_sep: `\";\"`. a string of desired SAM attributes, in the order desired for the output SAM. Tags can be listed in any combination/order.\n\n***Presets:\n- None ... no attributes\n- Standard ... NH HI AS nM\n- All ... NH HI AS nM NM MD jM jI MC ch\n***Alignment:\n- NH ... number of loci the reads maps to: =1 for unique mappers, \u003e1 for multimappers. Standard SAM tag.\n- HI ... multiple alignment index, starts with --out_sam_attr_ih_start (=1 by default). Standard SAM tag.\n- AS ... local alignment score, +1/-1 for matches/mismateches, score* penalties for indels and gaps. For PE reads, total score for two mates. Stadnard SAM tag.\n- nM ... number of mismatches. For PE reads, sum over two mates.\n- NM ... edit distance to the reference (number of mismatched + inserted + deleted bases) for each mate. Standard SAM tag.\n- MD ... string encoding mismatched and deleted reference bases (see standard SAM specifications). Standard SAM tag.\n- jM ... intron motifs for all junctions (i.e. N in CIGAR): 0: non-canonical; 1: GT/AG, 2: CT/AC, 3: GC/AG, 4: CT/GC, 5: AT/AC, 6: GT/AT. If splice junctions database is used, and a junction is annotated, 20 is added to its motif value.\n- jI ... start and end of introns for all junctions (1-based).\n- XS ... alignment strand according to --out_sam_strand_field.\n- MC ... mate\u0027s CIGAR string. Standard SAM tag.\n- ch ... marks all segment of all chimeric alingments for --chim_out_type WithinBAM output.\n- cN ... number of bases clipped from the read ends: 5\u0027 and 3\u0027\n***Variation:\n- vA ... variant allele\n- vG ... genomic coordinate of the variant overlapped by the read.\n- vW ... 1 - alignment passes WASP filtering; 2,3,4,5,6,7 - alignment does not pass WASP filtering. Requires --wasp_output_mode SAMtag.\n- ha ... haplotype (1/2) when mapping to the diploid genome. Requires genome generated with --genomeTransformType Diploid .\n***STARsolo:\n- CR CY UR UY ... sequences and quality scores of cell barcodes and UMIs for the solo* demultiplexing.\n- GX GN ... gene ID and gene name for unique-gene reads.\n- gx gn ... gene IDs and gene names for unique- and multi-gene reads.\n- CB UB ... error-corrected cell barcodes and UMIs for solo* demultiplexing. Requires --out_sam_type BAM SortedByCoordinate.\n- sM ... assessment of CB and UMI.\n- sS ... sequence of the entire barcode (CB,UMI,adapter).\n- sQ ... quality of the entire barcode.\n- sF ... type of feature overlap and number of features for each alignment\n***Unsupported/undocumented:\n- rB ... alignment block read/genomic coordinates.\n- vR ... read coordinate of the variant." + + } + + + , + "out_sam_attr_ih_start": { + "type": + "integer", + "description": "Type: `integer`, example: `1`. start value for the IH attribute", + "help_text": "Type: `integer`, example: `1`. start value for the IH attribute. 0 may be required by some downstream software, such as Cufflinks or StringTie." + + } + + + , + "out_sam_unmapped": { + "type": + "string", + "description": "Type: List of `string`, multiple_sep: `\";\"`. output of unmapped reads in the SAM format\n\n1st word:\n- None ", + "help_text": "Type: List of `string`, multiple_sep: `\";\"`. output of unmapped reads in the SAM format\n\n1st word:\n- None ... no output\n- Within ... output unmapped reads within the main SAM file (i.e. Aligned.out.sam)\n2nd word:\n- KeepPairs ... record unmapped mate for each alignment, and, in case of unsorted output, keep it adjacent to its mapped mate. Only affects multi-mapping reads." + + } + + + , + "out_sam_order": { + "type": + "string", + "description": "Type: `string`, example: `Paired`. type of sorting for the SAM output\n\nPaired: one mate after the other for all paired alignments\nPairedKeepInputOrder: one mate after the other for all paired alignments, the order is kept the same as in the input FASTQ files", + "help_text": "Type: `string`, example: `Paired`. type of sorting for the SAM output\n\nPaired: one mate after the other for all paired alignments\nPairedKeepInputOrder: one mate after the other for all paired alignments, the order is kept the same as in the input FASTQ files" + + } + + + , + "out_sam_primary_flag": { + "type": + "string", + "description": "Type: `string`, example: `OneBestScore`. which alignments are considered primary - all others will be marked with 0x100 bit in the FLAG\n\n- OneBestScore ", + "help_text": "Type: `string`, example: `OneBestScore`. which alignments are considered primary - all others will be marked with 0x100 bit in the FLAG\n\n- OneBestScore ... only one alignment with the best score is primary\n- AllBestScore ... all alignments with the best score are primary" + + } + + + , + "out_sam_read_id": { + "type": + "string", + "description": "Type: `string`, example: `Standard`. read ID record type\n\n- Standard ", + "help_text": "Type: `string`, example: `Standard`. read ID record type\n\n- Standard ... first word (until space) from the FASTx read ID line, removing /1,/2 from the end\n- Number ... read number (index) in the FASTx file" + + } + + + , + "out_sam_mapq_unique": { + "type": + "integer", + "description": "Type: `integer`, example: `255`. 0 to 255: the MAPQ value for unique mappers", + "help_text": "Type: `integer`, example: `255`. 0 to 255: the MAPQ value for unique mappers" + + } + + + , + "out_sam_flag_or": { + "type": + "integer", + "description": "Type: `integer`, example: `0`. 0 to 65535: sam FLAG will be bitwise OR\u0027d with this value, i", + "help_text": "Type: `integer`, example: `0`. 0 to 65535: sam FLAG will be bitwise OR\u0027d with this value, i.e. FLAG=FLAG | outSAMflagOR. This is applied after all flags have been set by STAR, and after outSAMflagAND. Can be used to set specific bits that are not set otherwise." + + } + + + , + "out_sam_flag_and": { + "type": + "integer", + "description": "Type: `integer`, example: `65535`. 0 to 65535: sam FLAG will be bitwise AND\u0027d with this value, i", + "help_text": "Type: `integer`, example: `65535`. 0 to 65535: sam FLAG will be bitwise AND\u0027d with this value, i.e. FLAG=FLAG \u0026 outSAMflagOR. This is applied after all flags have been set by STAR, but before outSAMflagOR. Can be used to unset specific bits that are not set otherwise." + + } + + + , + "out_sam_attr_rg_line": { + "type": + "string", + "description": "Type: List of `string`, multiple_sep: `\";\"`. SAM/BAM read group line", + "help_text": "Type: List of `string`, multiple_sep: `\";\"`. SAM/BAM read group line. The first word contains the read group identifier and must start with \"ID:\", e.g. --out_sam_attr_rg_line ID:xxx CN:yy \"DS:z z z\".\n\nxxx will be added as RG tag to each output alignment. Any spaces in the tag values have to be double quoted.\nComma separated RG lines correspons to different (comma separated) input files in --readFilesIn. Commas have to be surrounded by spaces, e.g.\n--out_sam_attr_rg_line ID:xxx , ID:zzz \"DS:z z\" , ID:yyy DS:yyyy" + + } + + + , + "out_sam_header_hd": { + "type": + "string", + "description": "Type: List of `string`, multiple_sep: `\";\"`. @HD (header) line of the SAM header", + "help_text": "Type: List of `string`, multiple_sep: `\";\"`. @HD (header) line of the SAM header" + + } + + + , + "out_sam_header_pg": { + "type": + "string", + "description": "Type: List of `string`, multiple_sep: `\";\"`. extra @PG (software) line of the SAM header (in addition to STAR)", + "help_text": "Type: List of `string`, multiple_sep: `\";\"`. extra @PG (software) line of the SAM header (in addition to STAR)" + + } + + + , + "out_sam_header_comment_file": { + "type": + "string", + "description": "Type: `string`. path to the file with @CO (comment) lines of the SAM header", + "help_text": "Type: `string`. path to the file with @CO (comment) lines of the SAM header" + + } + + + , + "out_sam_filter": { + "type": + "string", + "description": "Type: List of `string`, multiple_sep: `\";\"`. filter the output into main SAM/BAM files\n\n- KeepOnlyAddedReferences ", + "help_text": "Type: List of `string`, multiple_sep: `\";\"`. filter the output into main SAM/BAM files\n\n- KeepOnlyAddedReferences ... only keep the reads for which all alignments are to the extra reference sequences added with --genome_fasta_files at the mapping stage.\n- KeepAllAddedReferences ... keep all alignments to the extra reference sequences added with --genome_fasta_files at the mapping stage." + + } + + + , + "out_sam_mult_nmax": { + "type": + "integer", + "description": "Type: `integer`, example: `-1`. max number of multiple alignments for a read that will be output to the SAM/BAM files", + "help_text": "Type: `integer`, example: `-1`. max number of multiple alignments for a read that will be output to the SAM/BAM files. Note that if this value is not equal to -1, the top scoring alignment will be output first\n\n- -1 ... all alignments (up to --out_filter_multimap_nmax) will be output" + + } + + + , + "out_sam_tlen": { + "type": + "integer", + "description": "Type: `integer`, example: `1`. calculation method for the TLEN field in the SAM/BAM files\n\n- 1 ", + "help_text": "Type: `integer`, example: `1`. calculation method for the TLEN field in the SAM/BAM files\n\n- 1 ... leftmost base of the (+)strand mate to rightmost base of the (-)mate. (+)sign for the (+)strand mate\n- 2 ... leftmost base of any mate to rightmost base of any mate. (+)sign for the mate with the leftmost base. This is different from 1 for overlapping mates with protruding ends" + + } + + + , + "out_bam_compression": { + "type": + "integer", + "description": "Type: `integer`, example: `1`. -1 to 10 BAM compression level, -1=default compression (6?), 0=no compression, 10=maximum compression", + "help_text": "Type: `integer`, example: `1`. -1 to 10 BAM compression level, -1=default compression (6?), 0=no compression, 10=maximum compression" + + } + + + , + "out_bam_sorting_thread_n": { + "type": + "integer", + "description": "Type: `integer`, example: `0`. \u003e=0: number of threads for BAM sorting", + "help_text": "Type: `integer`, example: `0`. \u003e=0: number of threads for BAM sorting. 0 will default to min(6,--runThreadN)." + + } + + + , + "out_bam_sorting_bins_n": { + "type": + "integer", + "description": "Type: `integer`, example: `50`. \u003e0: number of genome bins for coordinate-sorting", + "help_text": "Type: `integer`, example: `50`. \u003e0: number of genome bins for coordinate-sorting" + + } + + +} +}, + + + "bam processing" : { + "title": "BAM processing", + "type": "object", + "description": "No description", + "properties": { + + + "bam_remove_duplicates_type": { + "type": + "string", + "description": "Type: `string`. mark duplicates in the BAM file, for now only works with (i) sorted BAM fed with inputBAMfile, and (ii) for paired-end alignments only\n\n- - ", + "help_text": "Type: `string`. mark duplicates in the BAM file, for now only works with (i) sorted BAM fed with inputBAMfile, and (ii) for paired-end alignments only\n\n- - ... no duplicate removal/marking\n- UniqueIdentical ... mark all multimappers, and duplicate unique mappers. The coordinates, FLAG, CIGAR must be identical\n- UniqueIdenticalNotMulti ... mark duplicate unique mappers but not multimappers." + + } + + + , + "bam_remove_duplicates_mate2bases_n": { + "type": + "integer", + "description": "Type: `integer`, example: `0`. number of bases from the 5\u0027 of mate 2 to use in collapsing (e", + "help_text": "Type: `integer`, example: `0`. number of bases from the 5\u0027 of mate 2 to use in collapsing (e.g. for RAMPAGE)" + + } + + +} +}, + + + "output wiggle" : { + "title": "Output Wiggle", + "type": "object", + "description": "No description", + "properties": { + + + "out_wig_type": { + "type": + "string", + "description": "Type: List of `string`, multiple_sep: `\";\"`. type of signal output, e", + "help_text": "Type: List of `string`, multiple_sep: `\";\"`. type of signal output, e.g. \"bedGraph\" OR \"bedGraph read1_5p\". Requires sorted BAM: --out_sam_type BAM SortedByCoordinate .\n\n1st word:\n- None ... no signal output\n- bedGraph ... bedGraph format\n- wiggle ... wiggle format\n2nd word:\n- read1_5p ... signal from only 5\u0027 of the 1st read, useful for CAGE/RAMPAGE etc\n- read2 ... signal from only 2nd read" + + } + + + , + "out_wig_strand": { + "type": + "string", + "description": "Type: `string`, example: `Stranded`. strandedness of wiggle/bedGraph output\n\n- Stranded ", + "help_text": "Type: `string`, example: `Stranded`. strandedness of wiggle/bedGraph output\n\n- Stranded ... separate strands, str1 and str2\n- Unstranded ... collapsed strands" + + } + + + , + "out_wig_references_prefix": { + "type": + "string", + "description": "Type: `string`. prefix matching reference names to include in the output wiggle file, e", + "help_text": "Type: `string`. prefix matching reference names to include in the output wiggle file, e.g. \"chr\", default \"-\" - include all references" + + } + + + , + "out_wig_norm": { + "type": + "string", + "description": "Type: `string`, example: `RPM`. type of normalization for the signal\n\n- RPM ", + "help_text": "Type: `string`, example: `RPM`. type of normalization for the signal\n\n- RPM ... reads per million of mapped reads\n- None ... no normalization, \"raw\" counts" + + } + + +} +}, + + + "output filtering" : { + "title": "Output Filtering", + "type": "object", + "description": "No description", + "properties": { + + + "out_filter_type": { + "type": + "string", + "description": "Type: `string`, example: `Normal`. type of filtering\n\n- Normal ", + "help_text": "Type: `string`, example: `Normal`. type of filtering\n\n- Normal ... standard filtering using only current alignment\n- BySJout ... keep only those reads that contain junctions that passed filtering into SJ.out.tab" + + } + + + , + "out_filter_multimap_score_range": { + "type": + "integer", + "description": "Type: `integer`, example: `1`. the score range below the maximum score for multimapping alignments", + "help_text": "Type: `integer`, example: `1`. the score range below the maximum score for multimapping alignments" + + } + + + , + "out_filter_multimap_nmax": { + "type": + "integer", + "description": "Type: `integer`, example: `10`. maximum number of loci the read is allowed to map to", + "help_text": "Type: `integer`, example: `10`. maximum number of loci the read is allowed to map to. Alignments (all of them) will be output only if the read maps to no more loci than this value.\n\nOtherwise no alignments will be output, and the read will be counted as \"mapped to too many loci\" in the Log.final.out ." + + } + + + , + "out_filter_mismatch_nmax": { + "type": + "integer", + "description": "Type: `integer`, example: `10`. alignment will be output only if it has no more mismatches than this value", + "help_text": "Type: `integer`, example: `10`. alignment will be output only if it has no more mismatches than this value." + + } + + + , + "out_filter_mismatch_nover_lmax": { + "type": + "number", + "description": "Type: `double`, example: `0.3`. alignment will be output only if its ratio of mismatches to *mapped* length is less than or equal to this value", + "help_text": "Type: `double`, example: `0.3`. alignment will be output only if its ratio of mismatches to *mapped* length is less than or equal to this value." + + } + + + , + "out_filter_mismatch_nover_read_lmax": { + "type": + "number", + "description": "Type: `double`, example: `1.0`. alignment will be output only if its ratio of mismatches to *read* length is less than or equal to this value", + "help_text": "Type: `double`, example: `1.0`. alignment will be output only if its ratio of mismatches to *read* length is less than or equal to this value." + + } + + + , + "out_filter_score_min": { + "type": + "integer", + "description": "Type: `integer`, example: `0`. alignment will be output only if its score is higher than or equal to this value", + "help_text": "Type: `integer`, example: `0`. alignment will be output only if its score is higher than or equal to this value." + + } + + + , + "out_filter_score_min_over_lread": { + "type": + "number", + "description": "Type: `double`, example: `0.66`. same as outFilterScoreMin, but normalized to read length (sum of mates\u0027 lengths for paired-end reads)", + "help_text": "Type: `double`, example: `0.66`. same as outFilterScoreMin, but normalized to read length (sum of mates\u0027 lengths for paired-end reads)" + + } + + + , + "out_filter_match_nmin": { + "type": + "integer", + "description": "Type: `integer`, example: `0`. alignment will be output only if the number of matched bases is higher than or equal to this value", + "help_text": "Type: `integer`, example: `0`. alignment will be output only if the number of matched bases is higher than or equal to this value." + + } + + + , + "out_filter_match_nmin_over_lread": { + "type": + "number", + "description": "Type: `double`, example: `0.66`. sam as outFilterMatchNmin, but normalized to the read length (sum of mates\u0027 lengths for paired-end reads)", + "help_text": "Type: `double`, example: `0.66`. sam as outFilterMatchNmin, but normalized to the read length (sum of mates\u0027 lengths for paired-end reads)." + + } + + + , + "out_filter_intron_motifs": { + "type": + "string", + "description": "Type: `string`. filter alignment using their motifs\n\n- None ", + "help_text": "Type: `string`. filter alignment using their motifs\n\n- None ... no filtering\n- RemoveNoncanonical ... filter out alignments that contain non-canonical junctions\n- RemoveNoncanonicalUnannotated ... filter out alignments that contain non-canonical unannotated junctions when using annotated splice junctions database. The annotated non-canonical junctions will be kept." + + } + + + , + "out_filter_intron_strands": { + "type": + "string", + "description": "Type: `string`, example: `RemoveInconsistentStrands`. filter alignments\n\n- RemoveInconsistentStrands ", + "help_text": "Type: `string`, example: `RemoveInconsistentStrands`. filter alignments\n\n- RemoveInconsistentStrands ... remove alignments that have junctions with inconsistent strands\n- None ... no filtering" + + } + + +} +}, + + + "output splice junctions (sj.out.tab)" : { + "title": "Output splice junctions (SJ.out.tab)", + "type": "object", + "description": "No description", + "properties": { + + + "out_sj_type": { + "type": + "string", + "description": "Type: `string`, example: `Standard`. type of splice junction output\n\n- Standard ", + "help_text": "Type: `string`, example: `Standard`. type of splice junction output\n\n- Standard ... standard SJ.out.tab output\n- None ... no splice junction output" + + } + + +} +}, + + + "output filtering: splice junctions" : { + "title": "Output Filtering: Splice Junctions", + "type": "object", + "description": "No description", + "properties": { + + + "out_sj_filter_reads": { + "type": + "string", + "description": "Type: `string`, example: `All`. which reads to consider for collapsed splice junctions output\n\n- All ", + "help_text": "Type: `string`, example: `All`. which reads to consider for collapsed splice junctions output\n\n- All ... all reads, unique- and multi-mappers\n- Unique ... uniquely mapping reads only" + + } + + + , + "out_sj_filter_overhang_min": { + "type": + "string", + "description": "Type: List of `integer`, example: `30;12;12;12`, multiple_sep: `\";\"`. minimum overhang length for splice junctions on both sides for: (1) non-canonical motifs, (2) GT/AG and CT/AC motif, (3) GC/AG and CT/GC motif, (4) AT/AC and GT/AT motif", + "help_text": "Type: List of `integer`, example: `30;12;12;12`, multiple_sep: `\";\"`. minimum overhang length for splice junctions on both sides for: (1) non-canonical motifs, (2) GT/AG and CT/AC motif, (3) GC/AG and CT/GC motif, (4) AT/AC and GT/AT motif. -1 means no output for that motif\n\ndoes not apply to annotated junctions" + + } + + + , + "out_sj_filter_count_unique_min": { + "type": + "string", + "description": "Type: List of `integer`, example: `3;1;1;1`, multiple_sep: `\";\"`. minimum uniquely mapping read count per junction for: (1) non-canonical motifs, (2) GT/AG and CT/AC motif, (3) GC/AG and CT/GC motif, (4) AT/AC and GT/AT motif", + "help_text": "Type: List of `integer`, example: `3;1;1;1`, multiple_sep: `\";\"`. minimum uniquely mapping read count per junction for: (1) non-canonical motifs, (2) GT/AG and CT/AC motif, (3) GC/AG and CT/GC motif, (4) AT/AC and GT/AT motif. -1 means no output for that motif\n\nJunctions are output if one of outSJfilterCountUniqueMin OR outSJfilterCountTotalMin conditions are satisfied\ndoes not apply to annotated junctions" + + } + + + , + "out_sj_filter_count_total_min": { + "type": + "string", + "description": "Type: List of `integer`, example: `3;1;1;1`, multiple_sep: `\";\"`. minimum total (multi-mapping+unique) read count per junction for: (1) non-canonical motifs, (2) GT/AG and CT/AC motif, (3) GC/AG and CT/GC motif, (4) AT/AC and GT/AT motif", + "help_text": "Type: List of `integer`, example: `3;1;1;1`, multiple_sep: `\";\"`. minimum total (multi-mapping+unique) read count per junction for: (1) non-canonical motifs, (2) GT/AG and CT/AC motif, (3) GC/AG and CT/GC motif, (4) AT/AC and GT/AT motif. -1 means no output for that motif\n\nJunctions are output if one of outSJfilterCountUniqueMin OR outSJfilterCountTotalMin conditions are satisfied\ndoes not apply to annotated junctions" + + } + + + , + "out_sj_filter_dist_to_other_sj_min": { + "type": + "string", + "description": "Type: List of `integer`, example: `10;0;5;10`, multiple_sep: `\";\"`. minimum allowed distance to other junctions\u0027 donor/acceptor\n\ndoes not apply to annotated junctions", + "help_text": "Type: List of `integer`, example: `10;0;5;10`, multiple_sep: `\";\"`. minimum allowed distance to other junctions\u0027 donor/acceptor\n\ndoes not apply to annotated junctions" + + } + + + , + "out_sj_filter_intron_max_vs_read_n": { + "type": + "string", + "description": "Type: List of `integer`, example: `50000;100000;200000`, multiple_sep: `\";\"`. maximum gap allowed for junctions supported by 1,2,3,,,N reads\n\ni", + "help_text": "Type: List of `integer`, example: `50000;100000;200000`, multiple_sep: `\";\"`. maximum gap allowed for junctions supported by 1,2,3,,,N reads\n\ni.e. by default junctions supported by 1 read can have gaps \u003c=50000b, by 2 reads: \u003c=100000b, by 3 reads: \u003c=200000. by \u003e=4 reads any gap \u003c=alignIntronMax\ndoes not apply to annotated junctions" + + } + + +} +}, + + + "scoring" : { + "title": "Scoring", + "type": "object", + "description": "No description", + "properties": { + + + "score_gap": { + "type": + "integer", + "description": "Type: `integer`, example: `0`. splice junction penalty (independent on intron motif)", + "help_text": "Type: `integer`, example: `0`. splice junction penalty (independent on intron motif)" + + } + + + , + "score_gap_noncan": { + "type": + "integer", + "description": "Type: `integer`, example: `-8`. non-canonical junction penalty (in addition to scoreGap)", + "help_text": "Type: `integer`, example: `-8`. non-canonical junction penalty (in addition to scoreGap)" + + } + + + , + "score_gap_gcag": { + "type": + "integer", + "description": "Type: `integer`, example: `-4`. GC/AG and CT/GC junction penalty (in addition to scoreGap)", + "help_text": "Type: `integer`, example: `-4`. GC/AG and CT/GC junction penalty (in addition to scoreGap)" + + } + + + , + "score_gap_atac": { + "type": + "integer", + "description": "Type: `integer`, example: `-8`. AT/AC and GT/AT junction penalty (in addition to scoreGap)", + "help_text": "Type: `integer`, example: `-8`. AT/AC and GT/AT junction penalty (in addition to scoreGap)" + + } + + + , + "score_genomic_length_log2scale": { + "type": + "integer", + "description": "Type: `integer`, example: `0`. extra score logarithmically scaled with genomic length of the alignment: scoreGenomicLengthLog2scale*log2(genomicLength)", + "help_text": "Type: `integer`, example: `0`. extra score logarithmically scaled with genomic length of the alignment: scoreGenomicLengthLog2scale*log2(genomicLength)" + + } + + + , + "score_del_open": { + "type": + "integer", + "description": "Type: `integer`, example: `-2`. deletion open penalty", + "help_text": "Type: `integer`, example: `-2`. deletion open penalty" + + } + + + , + "score_del_base": { + "type": + "integer", + "description": "Type: `integer`, example: `-2`. deletion extension penalty per base (in addition to scoreDelOpen)", + "help_text": "Type: `integer`, example: `-2`. deletion extension penalty per base (in addition to scoreDelOpen)" + + } + + + , + "score_ins_open": { + "type": + "integer", + "description": "Type: `integer`, example: `-2`. insertion open penalty", + "help_text": "Type: `integer`, example: `-2`. insertion open penalty" + + } + + + , + "score_ins_base": { + "type": + "integer", + "description": "Type: `integer`, example: `-2`. insertion extension penalty per base (in addition to scoreInsOpen)", + "help_text": "Type: `integer`, example: `-2`. insertion extension penalty per base (in addition to scoreInsOpen)" + + } + + + , + "score_stitch_sj_shift": { + "type": + "integer", + "description": "Type: `integer`, example: `1`. maximum score reduction while searching for SJ boundaries in the stitching step", + "help_text": "Type: `integer`, example: `1`. maximum score reduction while searching for SJ boundaries in the stitching step" + + } + + +} +}, + + + "alignments and seeding" : { + "title": "Alignments and Seeding", + "type": "object", + "description": "No description", + "properties": { + + + "seed_search_start_lmax": { + "type": + "integer", + "description": "Type: `integer`, example: `50`. defines the search start point through the read - the read is split into pieces no longer than this value", + "help_text": "Type: `integer`, example: `50`. defines the search start point through the read - the read is split into pieces no longer than this value" + + } + + + , + "seed_search_start_lmax_over_lread": { + "type": + "number", + "description": "Type: `double`, example: `1.0`. seedSearchStartLmax normalized to read length (sum of mates\u0027 lengths for paired-end reads)", + "help_text": "Type: `double`, example: `1.0`. seedSearchStartLmax normalized to read length (sum of mates\u0027 lengths for paired-end reads)" + + } + + + , + "seed_search_lmax": { + "type": + "integer", + "description": "Type: `integer`, example: `0`. defines the maximum length of the seeds, if =0 seed length is not limited", + "help_text": "Type: `integer`, example: `0`. defines the maximum length of the seeds, if =0 seed length is not limited" + + } + + + , + "seed_multimap_nmax": { + "type": + "integer", + "description": "Type: `integer`, example: `10000`. only pieces that map fewer than this value are utilized in the stitching procedure", + "help_text": "Type: `integer`, example: `10000`. only pieces that map fewer than this value are utilized in the stitching procedure" + + } + + + , + "seed_per_read_nmax": { + "type": + "integer", + "description": "Type: `integer`, example: `1000`. max number of seeds per read", + "help_text": "Type: `integer`, example: `1000`. max number of seeds per read" + + } + + + , + "seed_per_window_nmax": { + "type": + "integer", + "description": "Type: `integer`, example: `50`. max number of seeds per window", + "help_text": "Type: `integer`, example: `50`. max number of seeds per window" + + } + + + , + "seed_none_loci_per_window": { + "type": + "integer", + "description": "Type: `integer`, example: `10`. max number of one seed loci per window", + "help_text": "Type: `integer`, example: `10`. max number of one seed loci per window" + + } + + + , + "seed_split_min": { + "type": + "integer", + "description": "Type: `integer`, example: `12`. min length of the seed sequences split by Ns or mate gap", + "help_text": "Type: `integer`, example: `12`. min length of the seed sequences split by Ns or mate gap" + + } + + + , + "seed_map_min": { + "type": + "integer", + "description": "Type: `integer`, example: `5`. min length of seeds to be mapped", + "help_text": "Type: `integer`, example: `5`. min length of seeds to be mapped" + + } + + + , + "align_intron_min": { + "type": + "integer", + "description": "Type: `integer`, example: `21`. minimum intron size, genomic gap is considered intron if its length\u003e=alignIntronMin, otherwise it is considered Deletion", + "help_text": "Type: `integer`, example: `21`. minimum intron size, genomic gap is considered intron if its length\u003e=alignIntronMin, otherwise it is considered Deletion" + + } + + + , + "align_intron_max": { + "type": + "integer", + "description": "Type: `integer`, example: `0`. maximum intron size, if 0, max intron size will be determined by (2^winBinNbits)*winAnchorDistNbins", + "help_text": "Type: `integer`, example: `0`. maximum intron size, if 0, max intron size will be determined by (2^winBinNbits)*winAnchorDistNbins" + + } + + + , + "align_mates_gap_max": { + "type": + "integer", + "description": "Type: `integer`, example: `0`. maximum gap between two mates, if 0, max intron gap will be determined by (2^winBinNbits)*winAnchorDistNbins", + "help_text": "Type: `integer`, example: `0`. maximum gap between two mates, if 0, max intron gap will be determined by (2^winBinNbits)*winAnchorDistNbins" + + } + + + , + "align_sj_overhang_min": { + "type": + "integer", + "description": "Type: `integer`, example: `5`. minimum overhang (i", + "help_text": "Type: `integer`, example: `5`. minimum overhang (i.e. block size) for spliced alignments" + + } + + + , + "align_sj_stitch_mismatch_nmax": { + "type": + "string", + "description": "Type: List of `integer`, example: `0;-1;0;0`, multiple_sep: `\";\"`. maximum number of mismatches for stitching of the splice junctions (-1: no limit)", + "help_text": "Type: List of `integer`, example: `0;-1;0;0`, multiple_sep: `\";\"`. maximum number of mismatches for stitching of the splice junctions (-1: no limit).\n\n(1) non-canonical motifs, (2) GT/AG and CT/AC motif, (3) GC/AG and CT/GC motif, (4) AT/AC and GT/AT motif." + + } + + + , + "align_sjdb_overhang_min": { + "type": + "integer", + "description": "Type: `integer`, example: `3`. minimum overhang (i", + "help_text": "Type: `integer`, example: `3`. minimum overhang (i.e. block size) for annotated (sjdb) spliced alignments" + + } + + + , + "align_spliced_mate_map_lmin": { + "type": + "integer", + "description": "Type: `integer`, example: `0`. minimum mapped length for a read mate that is spliced", + "help_text": "Type: `integer`, example: `0`. minimum mapped length for a read mate that is spliced" + + } + + + , + "align_spliced_mate_map_lmin_over_lmate": { + "type": + "number", + "description": "Type: `double`, example: `0.66`. alignSplicedMateMapLmin normalized to mate length", + "help_text": "Type: `double`, example: `0.66`. alignSplicedMateMapLmin normalized to mate length" + + } + + + , + "align_windows_per_read_nmax": { + "type": + "integer", + "description": "Type: `integer`, example: `10000`. max number of windows per read", + "help_text": "Type: `integer`, example: `10000`. max number of windows per read" + + } + + + , + "align_transcripts_per_window_nmax": { + "type": + "integer", + "description": "Type: `integer`, example: `100`. max number of transcripts per window", + "help_text": "Type: `integer`, example: `100`. max number of transcripts per window" + + } + + + , + "align_transcripts_per_read_nmax": { + "type": + "integer", + "description": "Type: `integer`, example: `10000`. max number of different alignments per read to consider", + "help_text": "Type: `integer`, example: `10000`. max number of different alignments per read to consider" + + } + + + , + "align_ends_type": { + "type": + "string", + "description": "Type: `string`, example: `Local`. type of read ends alignment\n\n- Local ", + "help_text": "Type: `string`, example: `Local`. type of read ends alignment\n\n- Local ... standard local alignment with soft-clipping allowed\n- EndToEnd ... force end-to-end read alignment, do not soft-clip\n- Extend5pOfRead1 ... fully extend only the 5p of the read1, all other ends: local alignment\n- Extend5pOfReads12 ... fully extend only the 5p of the both read1 and read2, all other ends: local alignment" + + } + + + , + "align_ends_protrude": { + "type": + "string", + "description": "Type: `string`, example: `0 ConcordantPair`. allow protrusion of alignment ends, i", + "help_text": "Type: `string`, example: `0 ConcordantPair`. allow protrusion of alignment ends, i.e. start (end) of the +strand mate downstream of the start (end) of the -strand mate\n\n1st word: int: maximum number of protrusion bases allowed\n2nd word: string:\n- ConcordantPair ... report alignments with non-zero protrusion as concordant pairs\n- DiscordantPair ... report alignments with non-zero protrusion as discordant pairs" + + } + + + , + "align_soft_clip_at_reference_ends": { + "type": + "string", + "description": "Type: `string`, example: `Yes`. allow the soft-clipping of the alignments past the end of the chromosomes\n\n- Yes ", + "help_text": "Type: `string`, example: `Yes`. allow the soft-clipping of the alignments past the end of the chromosomes\n\n- Yes ... allow\n- No ... prohibit, useful for compatibility with Cufflinks" + + } + + + , + "align_insertion_flush": { + "type": + "string", + "description": "Type: `string`. how to flush ambiguous insertion positions\n\n- None ", + "help_text": "Type: `string`. how to flush ambiguous insertion positions\n\n- None ... insertions are not flushed\n- Right ... insertions are flushed to the right" + + } + + +} +}, + + + "paired-end reads" : { + "title": "Paired-End reads", + "type": "object", + "description": "No description", + "properties": { + + + "pe_overlap_nbases_min": { + "type": + "integer", + "description": "Type: `integer`, example: `0`. minimum number of overlapping bases to trigger mates merging and realignment", + "help_text": "Type: `integer`, example: `0`. minimum number of overlapping bases to trigger mates merging and realignment. Specify \u003e0 value to switch on the \"merginf of overlapping mates\" algorithm." + + } + + + , + "pe_overlap_mm_p": { + "type": + "number", + "description": "Type: `double`, example: `0.01`. maximum proportion of mismatched bases in the overlap area", + "help_text": "Type: `double`, example: `0.01`. maximum proportion of mismatched bases in the overlap area" + + } + + +} +}, + + + "windows, anchors, binning" : { + "title": "Windows, Anchors, Binning", + "type": "object", + "description": "No description", + "properties": { + + + "win_anchor_multimap_nmax": { + "type": + "integer", + "description": "Type: `integer`, example: `50`. max number of loci anchors are allowed to map to", + "help_text": "Type: `integer`, example: `50`. max number of loci anchors are allowed to map to" + + } + + + , + "win_bin_nbits": { + "type": + "integer", + "description": "Type: `integer`, example: `16`. =log2(winBin), where winBin is the size of the bin for the windows/clustering, each window will occupy an integer number of bins", + "help_text": "Type: `integer`, example: `16`. =log2(winBin), where winBin is the size of the bin for the windows/clustering, each window will occupy an integer number of bins." + + } + + + , + "win_anchor_dist_nbins": { + "type": + "integer", + "description": "Type: `integer`, example: `9`. max number of bins between two anchors that allows aggregation of anchors into one window", + "help_text": "Type: `integer`, example: `9`. max number of bins between two anchors that allows aggregation of anchors into one window" + + } + + + , + "win_flank_nbins": { + "type": + "integer", + "description": "Type: `integer`, example: `4`. log2(winFlank), where win Flank is the size of the left and right flanking regions for each window", + "help_text": "Type: `integer`, example: `4`. log2(winFlank), where win Flank is the size of the left and right flanking regions for each window" + + } + + + , + "win_read_coverage_relative_min": { + "type": + "number", + "description": "Type: `double`, example: `0.5`. minimum relative coverage of the read sequence by the seeds in a window, for STARlong algorithm only", + "help_text": "Type: `double`, example: `0.5`. minimum relative coverage of the read sequence by the seeds in a window, for STARlong algorithm only." + + } + + + , + "win_read_coverage_bases_min": { + "type": + "integer", + "description": "Type: `integer`, example: `0`. minimum number of bases covered by the seeds in a window , for STARlong algorithm only", + "help_text": "Type: `integer`, example: `0`. minimum number of bases covered by the seeds in a window , for STARlong algorithm only." + + } + + +} +}, + + + "chimeric alignments" : { + "title": "Chimeric Alignments", + "type": "object", + "description": "No description", + "properties": { + + + "chim_out_type": { + "type": + "string", + "description": "Type: List of `string`, example: `Junctions`, multiple_sep: `\";\"`. type of chimeric output\n\n- Junctions ", + "help_text": "Type: List of `string`, example: `Junctions`, multiple_sep: `\";\"`. type of chimeric output\n\n- Junctions ... Chimeric.out.junction\n- SeparateSAMold ... output old SAM into separate Chimeric.out.sam file\n- WithinBAM ... output into main aligned BAM files (Aligned.*.bam)\n- WithinBAM HardClip ... (default) hard-clipping in the CIGAR for supplemental chimeric alignments (default if no 2nd word is present)\n- WithinBAM SoftClip ... soft-clipping in the CIGAR for supplemental chimeric alignments" + + } + + + , + "chim_segment_min": { + "type": + "integer", + "description": "Type: `integer`, example: `0`. minimum length of chimeric segment length, if ==0, no chimeric output", + "help_text": "Type: `integer`, example: `0`. minimum length of chimeric segment length, if ==0, no chimeric output" + + } + + + , + "chim_score_min": { + "type": + "integer", + "description": "Type: `integer`, example: `0`. minimum total (summed) score of the chimeric segments", + "help_text": "Type: `integer`, example: `0`. minimum total (summed) score of the chimeric segments" + + } + + + , + "chim_score_drop_max": { + "type": + "integer", + "description": "Type: `integer`, example: `20`. max drop (difference) of chimeric score (the sum of scores of all chimeric segments) from the read length", + "help_text": "Type: `integer`, example: `20`. max drop (difference) of chimeric score (the sum of scores of all chimeric segments) from the read length" + + } + + + , + "chim_score_separation": { + "type": + "integer", + "description": "Type: `integer`, example: `10`. minimum difference (separation) between the best chimeric score and the next one", + "help_text": "Type: `integer`, example: `10`. minimum difference (separation) between the best chimeric score and the next one" + + } + + + , + "chim_score_junction_non_gtag": { + "type": + "integer", + "description": "Type: `integer`, example: `-1`. penalty for a non-GT/AG chimeric junction", + "help_text": "Type: `integer`, example: `-1`. penalty for a non-GT/AG chimeric junction" + + } + + + , + "chim_junction_overhang_min": { + "type": + "integer", + "description": "Type: `integer`, example: `20`. minimum overhang for a chimeric junction", + "help_text": "Type: `integer`, example: `20`. minimum overhang for a chimeric junction" + + } + + + , + "chim_segment_read_gap_max": { + "type": + "integer", + "description": "Type: `integer`, example: `0`. maximum gap in the read sequence between chimeric segments", + "help_text": "Type: `integer`, example: `0`. maximum gap in the read sequence between chimeric segments" + + } + + + , + "chim_filter": { + "type": + "string", + "description": "Type: List of `string`, example: `banGenomicN`, multiple_sep: `\";\"`. different filters for chimeric alignments\n\n- None ", + "help_text": "Type: List of `string`, example: `banGenomicN`, multiple_sep: `\";\"`. different filters for chimeric alignments\n\n- None ... no filtering\n- banGenomicN ... Ns are not allowed in the genome sequence around the chimeric junction" + + } + + + , + "chim_main_segment_mult_nmax": { + "type": + "integer", + "description": "Type: `integer`, example: `10`. maximum number of multi-alignments for the main chimeric segment", + "help_text": "Type: `integer`, example: `10`. maximum number of multi-alignments for the main chimeric segment. =1 will prohibit multimapping main segments." + + } + + + , + "chim_multimap_nmax": { + "type": + "integer", + "description": "Type: `integer`, example: `0`. maximum number of chimeric multi-alignments\n\n- 0 ", + "help_text": "Type: `integer`, example: `0`. maximum number of chimeric multi-alignments\n\n- 0 ... use the old scheme for chimeric detection which only considered unique alignments" + + } + + + , + "chim_multimap_score_range": { + "type": + "integer", + "description": "Type: `integer`, example: `1`. the score range for multi-mapping chimeras below the best chimeric score", + "help_text": "Type: `integer`, example: `1`. the score range for multi-mapping chimeras below the best chimeric score. Only works with --chim_multimap_nmax \u003e 1" + + } + + + , + "chim_nonchim_score_drop_min": { + "type": + "integer", + "description": "Type: `integer`, example: `20`. to trigger chimeric detection, the drop in the best non-chimeric alignment score with respect to the read length has to be greater than this value", + "help_text": "Type: `integer`, example: `20`. to trigger chimeric detection, the drop in the best non-chimeric alignment score with respect to the read length has to be greater than this value" + + } + + + , + "chim_out_junction_format": { + "type": + "integer", + "description": "Type: `integer`, example: `0`. formatting type for the Chimeric", + "help_text": "Type: `integer`, example: `0`. formatting type for the Chimeric.out.junction file\n\n- 0 ... no comment lines/headers\n- 1 ... comment lines at the end of the file: command line and Nreads: total, unique/multi-mapping" + + } + + +} +}, + + + "quantification of annotations" : { + "title": "Quantification of Annotations", + "type": "object", + "description": "No description", + "properties": { + + + "quant_mode": { + "type": + "string", + "description": "Type: List of `string`, multiple_sep: `\";\"`. types of quantification requested\n\n- - ", + "help_text": "Type: List of `string`, multiple_sep: `\";\"`. types of quantification requested\n\n- - ... none\n- TranscriptomeSAM ... output SAM/BAM alignments to transcriptome into a separate file\n- GeneCounts ... count reads per gene" + + } + + + , + "quant_transcriptome_bam_compression": { + "type": + "integer", + "description": "Type: `integer`, example: `1`. -2 to 10 transcriptome BAM compression level\n\n- -2 ", + "help_text": "Type: `integer`, example: `1`. -2 to 10 transcriptome BAM compression level\n\n- -2 ... no BAM output\n- -1 ... default compression (6?)\n- 0 ... no compression\n- 10 ... maximum compression" + + } + + + , + "quant_transcriptome_sam_output": { + "type": + "string", + "description": "Type: `string`, example: `BanSingleEnd_BanIndels_ExtendSoftclip`. alignment filtering for TranscriptomeSAM output\n\n- BanSingleEnd_BanIndels_ExtendSoftclip ", + "help_text": "Type: `string`, example: `BanSingleEnd_BanIndels_ExtendSoftclip`. alignment filtering for TranscriptomeSAM output\n\n- BanSingleEnd_BanIndels_ExtendSoftclip ... prohibit indels and single-end alignments, extend softclips - compatible with RSEM\n- BanSingleEnd ... prohibit single-end alignments, allow indels and softclips\n- BanSingleEnd_ExtendSoftclip ... prohibit single-end alignments, extend softclips, allow indels" + + } + + +} +}, + + + "2-pass mapping" : { + "title": "2-pass Mapping", + "type": "object", + "description": "No description", + "properties": { + + + "twopass_mode": { + "type": + "string", + "description": "Type: `string`. 2-pass mapping mode", + "help_text": "Type: `string`. 2-pass mapping mode.\n\n- None ... 1-pass mapping\n- Basic ... basic 2-pass mapping, with all 1st pass junctions inserted into the genome indices on the fly" + + } + + + , + "twopass1reads_n": { + "type": + "integer", + "description": "Type: `integer`, example: `-1`. number of reads to process for the 1st step", + "help_text": "Type: `integer`, example: `-1`. number of reads to process for the 1st step. Use very large number (or default -1) to map all reads in the first step." + + } + + +} +}, + + + "wasp parameters" : { + "title": "WASP parameters", + "type": "object", + "description": "No description", + "properties": { + + + "wasp_output_mode": { + "type": + "string", + "description": "Type: `string`. WASP allele-specific output type", + "help_text": "Type: `string`. WASP allele-specific output type. This is re-implementation of the original WASP mappability filtering by Bryce van de Geijn, Graham McVicker, Yoav Gilad \u0026 Jonathan K Pritchard. Please cite the original WASP paper: Nature Methods 12, 1061-1063 (2015), https://www.nature.com/articles/nmeth.3582 .\n\n- SAMtag ... add WASP tags to the alignments that pass WASP filtering" + + } + + +} +}, + + + "starsolo (single cell rna-seq) parameters" : { + "title": "STARsolo (single cell RNA-seq) parameters", + "type": "object", + "description": "No description", + "properties": { + + + "solo_type": { + "type": + "string", + "description": "Type: List of `string`, multiple_sep: `\";\"`. type of single-cell RNA-seq\n\n- CB_UMI_Simple ", + "help_text": "Type: List of `string`, multiple_sep: `\";\"`. type of single-cell RNA-seq\n\n- CB_UMI_Simple ... (a.k.a. Droplet) one UMI and one Cell Barcode of fixed length in read2, e.g. Drop-seq and 10X Chromium.\n- CB_UMI_Complex ... multiple Cell Barcodes of varying length, one UMI of fixed length and one adapter sequence of fixed length are allowed in read2 only (e.g. inDrop, ddSeq).\n- CB_samTagOut ... output Cell Barcode as CR and/or CB SAm tag. No UMI counting. --readFilesIn cDNA_read1 [cDNA_read2 if paired-end] CellBarcode_read . Requires --out_sam_type BAM Unsorted [and/or SortedByCoordinate]\n- SmartSeq ... Smart-seq: each cell in a separate FASTQ (paired- or single-end), barcodes are corresponding read-groups, no UMI sequences, alignments deduplicated according to alignment start and end (after extending soft-clipped bases)" + + } + + + , + "solo_cb_type": { + "type": + "string", + "description": "Type: `string`, example: `Sequence`. cell barcode type\n\nSequence: cell barcode is a sequence (standard option)\nString: cell barcode is an arbitrary string", + "help_text": "Type: `string`, example: `Sequence`. cell barcode type\n\nSequence: cell barcode is a sequence (standard option)\nString: cell barcode is an arbitrary string" + + } + + + , + "solo_cb_whitelist": { + "type": + "string", + "description": "Type: List of `string`, multiple_sep: `\";\"`. file(s) with whitelist(s) of cell barcodes", + "help_text": "Type: List of `string`, multiple_sep: `\";\"`. file(s) with whitelist(s) of cell barcodes. Only --solo_type CB_UMI_Complex allows more than one whitelist file.\n\n- None ... no whitelist: all cell barcodes are allowed" + + } + + + , + "solo_cb_start": { + "type": + "integer", + "description": "Type: `integer`, example: `1`. cell barcode start base", + "help_text": "Type: `integer`, example: `1`. cell barcode start base" + + } + + + , + "solo_cb_len": { + "type": + "integer", + "description": "Type: `integer`, example: `16`. cell barcode length", + "help_text": "Type: `integer`, example: `16`. cell barcode length" + + } + + + , + "solo_umi_start": { + "type": + "integer", + "description": "Type: `integer`, example: `17`. UMI start base", + "help_text": "Type: `integer`, example: `17`. UMI start base" + + } + + + , + "solo_umi_len": { + "type": + "integer", + "description": "Type: `integer`, example: `10`. UMI length", + "help_text": "Type: `integer`, example: `10`. UMI length" + + } + + + , + "solo_barcode_read_length": { + "type": + "integer", + "description": "Type: `integer`, example: `1`. length of the barcode read\n\n- 1 ", + "help_text": "Type: `integer`, example: `1`. length of the barcode read\n\n- 1 ... equal to sum of soloCBlen+soloUMIlen\n- 0 ... not defined, do not check" + + } + + + , + "solo_barcode_mate": { + "type": + "integer", + "description": "Type: `integer`, example: `0`. identifies which read mate contains the barcode (CB+UMI) sequence\n\n- 0 ", + "help_text": "Type: `integer`, example: `0`. identifies which read mate contains the barcode (CB+UMI) sequence\n\n- 0 ... barcode sequence is on separate read, which should always be the last file in the --readFilesIn listed\n- 1 ... barcode sequence is a part of mate 1\n- 2 ... barcode sequence is a part of mate 2" + + } + + + , + "solo_cb_position": { + "type": + "string", + "description": "Type: List of `string`, multiple_sep: `\";\"`. position of Cell Barcode(s) on the barcode read", + "help_text": "Type: List of `string`, multiple_sep: `\";\"`. position of Cell Barcode(s) on the barcode read.\n\nPresently only works with --solo_type CB_UMI_Complex, and barcodes are assumed to be on Read2.\nFormat for each barcode: startAnchor_startPosition_endAnchor_endPosition\nstart(end)Anchor defines the Anchor Base for the CB: 0: read start; 1: read end; 2: adapter start; 3: adapter end\nstart(end)Position is the 0-based position with of the CB start(end) with respect to the Anchor Base\nString for different barcodes are separated by space.\nExample: inDrop (Zilionis et al, Nat. Protocols, 2017):\n--solo_cb_position 0_0_2_-1 3_1_3_8" + + } + + + , + "solo_umi_position": { + "type": + "string", + "description": "Type: `string`. position of the UMI on the barcode read, same as soloCBposition\n\nExample: inDrop (Zilionis et al, Nat", + "help_text": "Type: `string`. position of the UMI on the barcode read, same as soloCBposition\n\nExample: inDrop (Zilionis et al, Nat. Protocols, 2017):\n--solo_cb_position 3_9_3_14" + + } + + + , + "solo_adapter_sequence": { + "type": + "string", + "description": "Type: `string`. adapter sequence to anchor barcodes", + "help_text": "Type: `string`. adapter sequence to anchor barcodes. Only one adapter sequence is allowed." + + } + + + , + "solo_adapter_mismatches_nmax": { + "type": + "integer", + "description": "Type: `integer`, example: `1`. maximum number of mismatches allowed in adapter sequence", + "help_text": "Type: `integer`, example: `1`. maximum number of mismatches allowed in adapter sequence." + + } + + + , + "solo_cb_match_wl_type": { + "type": + "string", + "description": "Type: `string`, example: `1MM_multi`. matching the Cell Barcodes to the WhiteList\n\n- Exact ", + "help_text": "Type: `string`, example: `1MM_multi`. matching the Cell Barcodes to the WhiteList\n\n- Exact ... only exact matches allowed\n- 1MM ... only one match in whitelist with 1 mismatched base allowed. Allowed CBs have to have at least one read with exact match.\n- 1MM_multi ... multiple matches in whitelist with 1 mismatched base allowed, posterior probability calculation is used choose one of the matches.\nAllowed CBs have to have at least one read with exact match. This option matches best with CellRanger 2.2.0\n- 1MM_multi_pseudocounts ... same as 1MM_Multi, but pseudocounts of 1 are added to all whitelist barcodes.\n- 1MM_multi_Nbase_pseudocounts ... same as 1MM_multi_pseudocounts, multimatching to WL is allowed for CBs with N-bases. This option matches best with CellRanger \u003e= 3.0.0\n- EditDist_2 ... allow up to edit distance of 3 fpr each of the barcodes. May include one deletion + one insertion. Only works with --solo_type CB_UMI_Complex. Matches to multiple passlist barcdoes are not allowed. Similar to ParseBio Split-seq pipeline." + + } + + + , + "solo_input_sam_attr_barcode_seq": { + "type": + "string", + "description": "Type: List of `string`, multiple_sep: `\";\"`. when inputting reads from a SAM file (--readsFileType SAM SE/PE), these SAM attributes mark the barcode sequence (in proper order)", + "help_text": "Type: List of `string`, multiple_sep: `\";\"`. when inputting reads from a SAM file (--readsFileType SAM SE/PE), these SAM attributes mark the barcode sequence (in proper order).\n\nFor instance, for 10X CellRanger or STARsolo BAMs, use --solo_input_sam_attr_barcode_seq CR UR .\nThis parameter is required when running STARsolo with input from SAM." + + } + + + , + "solo_input_sam_attr_barcode_qual": { + "type": + "string", + "description": "Type: List of `string`, multiple_sep: `\";\"`. when inputting reads from a SAM file (--readsFileType SAM SE/PE), these SAM attributes mark the barcode qualities (in proper order)", + "help_text": "Type: List of `string`, multiple_sep: `\";\"`. when inputting reads from a SAM file (--readsFileType SAM SE/PE), these SAM attributes mark the barcode qualities (in proper order).\n\nFor instance, for 10X CellRanger or STARsolo BAMs, use --solo_input_sam_attr_barcode_qual CY UY .\nIf this parameter is \u0027-\u0027 (default), the quality \u0027H\u0027 will be assigned to all bases." + + } + + + , + "solo_strand": { + "type": + "string", + "description": "Type: `string`, example: `Forward`. strandedness of the solo libraries:\n\n- Unstranded ", + "help_text": "Type: `string`, example: `Forward`. strandedness of the solo libraries:\n\n- Unstranded ... no strand information\n- Forward ... read strand same as the original RNA molecule\n- Reverse ... read strand opposite to the original RNA molecule" + + } + + + , + "solo_features": { + "type": + "string", + "description": "Type: List of `string`, example: `Gene`, multiple_sep: `\";\"`. genomic features for which the UMI counts per Cell Barcode are collected\n\n- Gene ", + "help_text": "Type: List of `string`, example: `Gene`, multiple_sep: `\";\"`. genomic features for which the UMI counts per Cell Barcode are collected\n\n- Gene ... genes: reads match the gene transcript\n- SJ ... splice junctions: reported in SJ.out.tab\n- GeneFull ... full gene (pre-mRNA): count all reads overlapping genes\u0027 exons and introns\n- GeneFull_ExonOverIntron ... full gene (pre-mRNA): count all reads overlapping genes\u0027 exons and introns: prioritize 100% overlap with exons\n- GeneFull_Ex50pAS ... full gene (pre-RNA): count all reads overlapping genes\u0027 exons and introns: prioritize \u003e50% overlap with exons. Do not count reads with 100% exonic overlap in the antisense direction." + + } + + + , + "solo_multi_mappers": { + "type": + "string", + "description": "Type: List of `string`, example: `Unique`, multiple_sep: `\";\"`. counting method for reads mapping to multiple genes\n\n- Unique ", + "help_text": "Type: List of `string`, example: `Unique`, multiple_sep: `\";\"`. counting method for reads mapping to multiple genes\n\n- Unique ... count only reads that map to unique genes\n- Uniform ... uniformly distribute multi-genic UMIs to all genes\n- Rescue ... distribute UMIs proportionally to unique+uniform counts (~ first iteration of EM)\n- PropUnique ... distribute UMIs proportionally to unique mappers, if present, and uniformly if not.\n- EM ... multi-gene UMIs are distributed using Expectation Maximization algorithm" + + } + + + , + "solo_umi_dedup": { + "type": + "string", + "description": "Type: List of `string`, example: `1MM_All`, multiple_sep: `\";\"`. type of UMI deduplication (collapsing) algorithm\n\n- 1MM_All ", + "help_text": "Type: List of `string`, example: `1MM_All`, multiple_sep: `\";\"`. type of UMI deduplication (collapsing) algorithm\n\n- 1MM_All ... all UMIs with 1 mismatch distance to each other are collapsed (i.e. counted once).\n- 1MM_Directional_UMItools ... follows the \"directional\" method from the UMI-tools by Smith, Heger and Sudbery (Genome Research 2017).\n- 1MM_Directional ... same as 1MM_Directional_UMItools, but with more stringent criteria for duplicate UMIs\n- Exact ... only exactly matching UMIs are collapsed.\n- NoDedup ... no deduplication of UMIs, count all reads.\n- 1MM_CR ... CellRanger2-4 algorithm for 1MM UMI collapsing." + + } + + + , + "solo_umi_filtering": { + "type": + "string", + "description": "Type: List of `string`, multiple_sep: `\";\"`. type of UMI filtering (for reads uniquely mapping to genes)\n\n- - ", + "help_text": "Type: List of `string`, multiple_sep: `\";\"`. type of UMI filtering (for reads uniquely mapping to genes)\n\n- - ... basic filtering: remove UMIs with N and homopolymers (similar to CellRanger 2.2.0).\n- MultiGeneUMI ... basic + remove lower-count UMIs that map to more than one gene.\n- MultiGeneUMI_All ... basic + remove all UMIs that map to more than one gene.\n- MultiGeneUMI_CR ... basic + remove lower-count UMIs that map to more than one gene, matching CellRanger \u003e 3.0.0 .\nOnly works with --solo_umi_dedup 1MM_CR" + + } + + + , + "solo_out_file_names": { + "type": + "string", + "description": "Type: List of `string`, example: `Solo.out/;features.tsv;barcodes.tsv;matrix.mtx`, multiple_sep: `\";\"`. file names for STARsolo output:\n\nfile_name_prefix gene_names barcode_sequences cell_feature_count_matrix", + "help_text": "Type: List of `string`, example: `Solo.out/;features.tsv;barcodes.tsv;matrix.mtx`, multiple_sep: `\";\"`. file names for STARsolo output:\n\nfile_name_prefix gene_names barcode_sequences cell_feature_count_matrix" + + } + + + , + "solo_cell_filter": { + "type": + "string", + "description": "Type: List of `string`, example: `CellRanger2.2;3000;0.99;10`, multiple_sep: `\";\"`. cell filtering type and parameters\n\n- None ", + "help_text": "Type: List of `string`, example: `CellRanger2.2;3000;0.99;10`, multiple_sep: `\";\"`. cell filtering type and parameters\n\n- None ... do not output filtered cells\n- TopCells ... only report top cells by UMI count, followed by the exact number of cells\n- CellRanger2.2 ... simple filtering of CellRanger 2.2.\nCan be followed by numbers: number of expected cells, robust maximum percentile for UMI count, maximum to minimum ratio for UMI count\nThe harcoded values are from CellRanger: nExpectedCells=3000; maxPercentile=0.99; maxMinRatio=10\n- EmptyDrops_CR ... EmptyDrops filtering in CellRanger flavor. Please cite the original EmptyDrops paper: A.T.L Lun et al, Genome Biology, 20, 63 (2019): https://genomebiology.biomedcentral.com/articles/10.1186/s13059-019-1662-y\nCan be followed by 10 numeric parameters: nExpectedCells maxPercentile maxMinRatio indMin indMax umiMin umiMinFracMedian candMaxN FDR simN\nThe harcoded values are from CellRanger: 3000 0.99 10 45000 90000 500 0.01 20000 0.01 10000" + + } + + + , + "solo_out_format_features_gene_field3": { + "type": + "string", + "description": "Type: List of `string`, example: `Gene Expression`, multiple_sep: `\";\"`. field 3 in the Gene features", + "help_text": "Type: List of `string`, example: `Gene Expression`, multiple_sep: `\";\"`. field 3 in the Gene features.tsv file. If \"-\", then no 3rd field is output." + + } + + + , + "solo_cell_read_stats": { + "type": + "string", + "description": "Type: `string`. Output reads statistics for each CB\n\n- Standard ", + "help_text": "Type: `string`. Output reads statistics for each CB\n\n- Standard ... standard output" + + } + + } }, @@ -175,6 +2215,98 @@ "$ref": "#/definitions/outputs" }, + { + "$ref": "#/definitions/run parameters" + }, + + { + "$ref": "#/definitions/genome parameters" + }, + + { + "$ref": "#/definitions/splice junctions database" + }, + + { + "$ref": "#/definitions/variation parameters" + }, + + { + "$ref": "#/definitions/read parameters" + }, + + { + "$ref": "#/definitions/read clipping" + }, + + { + "$ref": "#/definitions/limits" + }, + + { + "$ref": "#/definitions/output: general" + }, + + { + "$ref": "#/definitions/output: sam and bam" + }, + + { + "$ref": "#/definitions/bam processing" + }, + + { + "$ref": "#/definitions/output wiggle" + }, + + { + "$ref": "#/definitions/output filtering" + }, + + { + "$ref": "#/definitions/output splice junctions (sj.out.tab)" + }, + + { + "$ref": "#/definitions/output filtering: splice junctions" + }, + + { + "$ref": "#/definitions/scoring" + }, + + { + "$ref": "#/definitions/alignments and seeding" + }, + + { + "$ref": "#/definitions/paired-end reads" + }, + + { + "$ref": "#/definitions/windows, anchors, binning" + }, + + { + "$ref": "#/definitions/chimeric alignments" + }, + + { + "$ref": "#/definitions/quantification of annotations" + }, + + { + "$ref": "#/definitions/2-pass mapping" + }, + + { + "$ref": "#/definitions/wasp parameters" + }, + + { + "$ref": "#/definitions/starsolo (single cell rna-seq) parameters" + }, + { "$ref": "#/definitions/nextflow input-output arguments" } diff --git a/target/nextflow/star/star_genome_generate/.config.vsh.yaml b/target/nextflow/star/star_genome_generate/.config.vsh.yaml index a3aab66a..b43b42a2 100644 --- a/target/nextflow/star/star_genome_generate/.config.vsh.yaml +++ b/target/nextflow/star/star_genome_generate/.config.vsh.yaml @@ -332,15 +332,15 @@ build_info: engine: "docker|native" output: "target/nextflow/star/star_genome_generate" executable: "target/nextflow/star/star_genome_generate/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/star/star_genome_generate/main.nf b/target/nextflow/star/star_genome_generate/main.nf index a6445ebc..c94a85f3 100644 --- a/target/nextflow/star/star_genome_generate/main.nf +++ b/target/nextflow/star/star_genome_generate/main.nf @@ -1,6 +1,6 @@ // star_genome_generate main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -763,8 +763,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -786,7 +789,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -824,8 +840,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1601,8 +1620,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2629,30 +2648,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2723,7 +2743,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -3168,15 +3188,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/star/star_genome_generate", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/star/star_genome_generate/nextflow_schema.json b/target/nextflow/star/star_genome_generate/nextflow_schema.json index cd35986c..60ab837b 100644 --- a/target/nextflow/star/star_genome_generate/nextflow_schema.json +++ b/target/nextflow/star/star_genome_generate/nextflow_schema.json @@ -17,8 +17,8 @@ "genome_fasta_files": { "type": "string", - "description": "Type: List of `file`, required, multiple_sep: `\":\"`. Path(s) to the fasta files with the genome sequences, separated by spaces", - "help_text": "Type: List of `file`, required, multiple_sep: `\":\"`. Path(s) to the fasta files with the genome sequences, separated by spaces. These files should be plain text FASTA files, they *cannot* be zipped.\n" + "description": "Type: List of `file`, required, multiple_sep: `\";\"`. Path(s) to the fasta files with the genome sequences, separated by spaces", + "help_text": "Type: List of `file`, required, multiple_sep: `\";\"`. Path(s) to the fasta files with the genome sequences, separated by spaces. These files should be plain text FASTA files, they *cannot* be zipped.\n" } @@ -87,8 +87,8 @@ "sjdb_gtf_tag_exon_parent_gene_name": { "type": "string", - "description": "Type: List of `string`, example: `gene_name`, multiple_sep: `\":\"`. GTF attribute name for parent gene name", - "help_text": "Type: List of `string`, example: `gene_name`, multiple_sep: `\":\"`. GTF attribute name for parent gene name" + "description": "Type: List of `string`, example: `gene_name`, multiple_sep: `\";\"`. GTF attribute name for parent gene name", + "help_text": "Type: List of `string`, example: `gene_name`, multiple_sep: `\";\"`. GTF attribute name for parent gene name" } @@ -97,8 +97,8 @@ "sjdb_gtf_tag_exon_parent_gene_type": { "type": "string", - "description": "Type: List of `string`, example: `gene_type:gene_biotype`, multiple_sep: `\":\"`. GTF attribute name for parent gene type", - "help_text": "Type: List of `string`, example: `gene_type:gene_biotype`, multiple_sep: `\":\"`. GTF attribute name for parent gene type" + "description": "Type: List of `string`, example: `gene_type;gene_biotype`, multiple_sep: `\";\"`. GTF attribute name for parent gene type", + "help_text": "Type: List of `string`, example: `gene_type;gene_biotype`, multiple_sep: `\";\"`. GTF attribute name for parent gene type" } diff --git a/target/nextflow/umi_tools/umi_tools_dedup/.config.vsh.yaml b/target/nextflow/umi_tools/umi_tools_dedup/.config.vsh.yaml index f5bbbfcb..6baa88d4 100644 --- a/target/nextflow/umi_tools/umi_tools_dedup/.config.vsh.yaml +++ b/target/nextflow/umi_tools/umi_tools_dedup/.config.vsh.yaml @@ -610,15 +610,15 @@ build_info: engine: "docker|native" output: "target/nextflow/umi_tools/umi_tools_dedup" executable: "target/nextflow/umi_tools/umi_tools_dedup/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/umi_tools/umi_tools_dedup/main.nf b/target/nextflow/umi_tools/umi_tools_dedup/main.nf index 6bc98626..350c9181 100644 --- a/target/nextflow/umi_tools/umi_tools_dedup/main.nf +++ b/target/nextflow/umi_tools/umi_tools_dedup/main.nf @@ -1,6 +1,6 @@ // umi_tools_dedup main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -763,8 +763,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -786,7 +789,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -824,8 +840,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1601,8 +1620,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2629,30 +2648,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2723,7 +2743,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -3460,15 +3480,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/umi_tools/umi_tools_dedup", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/umi_tools/umi_tools_extract/.config.vsh.yaml b/target/nextflow/umi_tools/umi_tools_extract/.config.vsh.yaml index 40e11658..7a21e682 100644 --- a/target/nextflow/umi_tools/umi_tools_extract/.config.vsh.yaml +++ b/target/nextflow/umi_tools/umi_tools_extract/.config.vsh.yaml @@ -448,15 +448,15 @@ build_info: engine: "docker|native" output: "target/nextflow/umi_tools/umi_tools_extract" executable: "target/nextflow/umi_tools/umi_tools_extract/main.nf" - viash_version: "0.9.0-RC6" - git_commit: "766ab6c9c3059004c7c3f205621909b2d8b0b26d" + viash_version: "0.9.0-RC7" + git_commit: "2b29a47575db9dbdff8448b287925c25d9a8b01d" git_remote: "https://github.com/viash-hub/biobox" package_config: name: "biobox" version: "main" description: "A collection of bioinformatics tools for working with sequence data.\n" info: null - viash_version: "0.9.0-RC6" + viash_version: "0.9.0-RC7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/umi_tools/umi_tools_extract/main.nf b/target/nextflow/umi_tools/umi_tools_extract/main.nf index 3198d42f..f72fb4e0 100644 --- a/target/nextflow/umi_tools/umi_tools_extract/main.nf +++ b/target/nextflow/umi_tools/umi_tools_extract/main.nf @@ -1,6 +1,6 @@ // umi_tools_extract main // -// This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a +// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a // derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from // Data Intuitive. // @@ -760,8 +760,11 @@ def runEach(Map args) { def fromState_ = args.fromState def toState_ = args.toState def filter_ = args.filter + def runIf_ = args.runIf def id_ = args.id + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + workflow runEachWf { take: input_ch main: @@ -783,7 +786,20 @@ def runEach(Map args) { [new_id] + tup.drop(1) } : filter_ch - def data_ch = id_ch | map{tup -> + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> def new_data = tup[1] if (fromState_ instanceof Map) { new_data = fromState_.collectEntries{ key0, key1 -> @@ -821,8 +837,11 @@ def runEach(Map args) { [tup[0], new_state] + tup.drop(3) } : out_ch + + def return_ch = post_ch + | concat(chPassthrough) - post_ch + return_ch } // mix all results @@ -1598,8 +1617,8 @@ def findStates(Map params, Map config) { // construct renameMap if (args.rename_keys) { def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(";") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey,newKey:oldKey'" + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" split } @@ -2626,30 +2645,31 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - def chModifiedFiltered = workflowArgs.filter ? - chModified | filter{workflowArgs.filter(it)} : - chModified def chRun = null def chPassthrough = null if (workflowArgs.runIf) { - def runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModified.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } chRun = runIfBranch.run chPassthrough = runIfBranch.passthrough } else { - chRun = chModifiedFiltered + chRun = chModified chPassthrough = Channel.empty() } + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + def chArgs = workflowArgs.fromState ? - chRun | map{ + chRunFiltered | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } : - chRun | map {tup -> tup.take(2)} + chRunFiltered | map {tup -> tup.take(2)} // fill in defaults def chArgsWithDefaults = chArgs @@ -2720,7 +2740,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -3289,15 +3309,15 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/umi_tools/umi_tools_extract", - "viash_version" : "0.9.0-RC6", - "git_commit" : "766ab6c9c3059004c7c3f205621909b2d8b0b26d", + "viash_version" : "0.9.0-RC7", + "git_commit" : "2b29a47575db9dbdff8448b287925c25d9a8b01d", "git_remote" : "https://github.com/viash-hub/biobox" }, "package_config" : { "name" : "biobox", "version" : "main", "description" : "A collection of bioinformatics tools for working with sequence data.\n", - "viash_version" : "0.9.0-RC6", + "viash_version" : "0.9.0-RC7", "source" : "src", "target" : "target", "config_mods" : [