From f00709ee19b5229313a967944ac5c4c7d32a9180 Mon Sep 17 00:00:00 2001 From: CI Date: Mon, 1 Jun 2026 12:59:55 +0000 Subject: [PATCH] Build branch craftbox/main with version main to craftbox on branch main (b74f552) Build pipeline: vsh-ci-build-template-xfb9n Source commit: https://github.com/viash-hub/craftbox/commit/b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1 Source message: Prepare for 0.3.2 release (#19) --- CHANGELOG.md | 18 ++ README.md | 12 +- README.qmd | 2 +- _viash.yaml | 2 +- src/concat_text/config.vsh.yaml | 13 +- src/concat_text/script.sh | 79 +++++-- src/concat_text/test.sh | 92 ++++++-- src/move_files_to_directory/config.vsh.yaml | 61 +++-- src/move_files_to_directory/script.sh | 37 +-- src/move_files_to_directory/test.sh | 18 +- .../check_disk_space/.config.vsh.yaml | 6 +- .../check_disk_space/check_disk_space | 6 +- .../executable/concat_text/.config.vsh.yaml | 33 ++- target/executable/concat_text/concat_text | 91 +++++--- target/executable/csv2fasta/.config.vsh.yaml | 6 +- target/executable/csv2fasta/csv2fasta | 6 +- .../move_files_to_directory/.config.vsh.yaml | 33 +-- .../move_files_to_directory | 73 +++--- .../sync_resources/.config.vsh.yaml | 6 +- .../executable/sync_resources/sync_resources | 6 +- target/executable/untar/.config.vsh.yaml | 6 +- target/executable/untar/untar | 6 +- .../check_disk_space/.config.vsh.yaml | 6 +- target/nextflow/check_disk_space/main.nf | 100 ++++---- target/nextflow/concat_text/.config.vsh.yaml | 33 ++- target/nextflow/concat_text/main.nf | 213 ++++++++++++------ target/nextflow/concat_text/nextflow.config | 2 +- .../nextflow/concat_text/nextflow_schema.json | 4 +- target/nextflow/csv2fasta/.config.vsh.yaml | 6 +- target/nextflow/csv2fasta/main.nf | 100 ++++---- .../move_files_to_directory/.config.vsh.yaml | 33 +-- .../nextflow/move_files_to_directory/main.nf | 169 ++++++++------ .../move_files_to_directory/nextflow.config | 2 +- .../nextflow_schema.json | 38 +++- .../nextflow/sync_resources/.config.vsh.yaml | 6 +- target/nextflow/sync_resources/main.nf | 100 ++++---- target/nextflow/untar/.config.vsh.yaml | 6 +- target/nextflow/untar/main.nf | 100 ++++---- 38 files changed, 943 insertions(+), 587 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2dcc638..8fc73ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,21 @@ +# craftbox 0.3.2 + +## NEW FEATURES + +* `move_files_to_directory`: add support for moving directories (PR #17). + +## MINOR CHANGES + +* Bump viash to 0.9.7 (PR #18). + +# craftbox 0.3.1 + +## MINOR CHANGES + +* `concat_text`: Allow input files to be a mixed bag of gzipped and plain text files (PR #12). + +* `concat_text`: Stream concatenated files to gzip (PR #12). + # craftbox 0.3.0 ## NEW FEATURES diff --git a/README.md b/README.md index d6187bf..726e87e 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ Viash components in craftbox can be run in various ways: ``` mermaid lang="mermaid flowchart TD - A[craftbox v0.3.0] --> B(Viash Hub Launch) + A[craftbox v0.3.2] --> B(Viash Hub Launch) A --> C(Viash CLI) A --> D(Nextflow CLI) A --> E(Seqera Cloud) @@ -53,7 +53,7 @@ flowchart TD ### 1. Via the Viash Hub Launch interface You can run this component directly from the Viash Hub [Launch -interface](https://www.viash-hub.com/launch?package=craftbox&version=v0.3.0&component=concat_rtext&runner=Executable). +interface](https://www.viash-hub.com/launch?package=craftbox&version=v0.3.2&component=concat_rtext&runner=Executable). ![](docs/viash-hub.png) @@ -63,9 +63,9 @@ You can run this component directly from the command line using the Viash CLI. ``` bash -viash run vsh://craftbox@v0.3.0/concat_rtext -- --help +viash run vsh://craftbox@v0.3.2/concat_rtext -- --help -viash run vsh://craftbox@v0.3.0/concat_rtext -- \ +viash run vsh://craftbox@v0.3.2/concat_rtext -- \ --input path/to/input.txt \ --input path/to/compressed.txt.gz \ --output path/to/output.txt @@ -80,7 +80,7 @@ You can run this component as a Nextflow pipeline. ``` bash nextflow run https://packages.viash-hub.com/vsh/craftbox \ - -revision v0.3.0 \ + -revision v0.3.2 \ -main-script target/nextflow/concat_rtext/main.nf \ -latest -resume \ -profile docker \ @@ -109,7 +109,7 @@ component as a dependency: ``` yaml dependencies: - name: concat_rtext - repository: vsh://craftbox@v0.3.0 + repository: vsh://craftbox@v0.3.2 ``` **Tip:** See the [Viash diff --git a/README.qmd b/README.qmd index 2660346..a2b4078 100644 --- a/README.qmd +++ b/README.qmd @@ -7,7 +7,7 @@ license <- paste0(package$links$repository, "/blob/main/LICENSE") contributing <- paste0(package$links$repository, "/blob/main/CONTRIBUTING.md") pkg <- package$name -ver <- if (!is.null(package$version)) package$version else "v0.3.0" +ver <- if (!is.null(package$version)) package$version else "v0.3.2" comp <- "concat_rtext" ``` # 🪡📦 `r pkg` diff --git a/_viash.yaml b/_viash.yaml index a8cb0fe..adfb90f 100644 --- a/_viash.yaml +++ b/_viash.yaml @@ -18,7 +18,7 @@ license: MIT links: issue_tracker: https://github.com/viash-hub/craftbox/issues repository: https://github.com/viash-hub/craftbox -viash_version: 0.9.4 +viash_version: 0.9.7 config_mods: | .requirements.commands := ['ps'] version: main diff --git a/src/concat_text/config.vsh.yaml b/src/concat_text/config.vsh.yaml index a76f83e..7a2c6a1 100644 --- a/src/concat_text/config.vsh.yaml +++ b/src/concat_text/config.vsh.yaml @@ -1,5 +1,5 @@ name: concat_text -summary: Concatenate a number of text files +summary: Concatenate multiple (possibly gzipped) text files description: | Concatenate a number of text files, handle gzipped text files gracefully and optionally gzip the output text file. @@ -11,12 +11,13 @@ authors: roles: [ author, maintainer ] - __merge__: /src/_authors/dries_schaumont.yaml roles: [ reviewer ] + - __merge__: /src/_authors/robrecht_cannoodt.yaml + roles: [ contributor ] info: improvements: | - This component could be improved in 2 ways: - 1. Allow for a mix of zipped and plain input files - 2. Allow to specify a compression algorithm for the output + This component could be improved: + 1. Allow to specify a compression algorithm for the output argument_groups: - name: Input arguments @@ -26,14 +27,14 @@ argument_groups: type: file multiple: true required: true - example: input?.txt.gz + example: input.txt.gz - name: Output arguments arguments: - name: "--gzip_output" type: boolean_true description: Should the output be zipped? - name: --output - description: File to write the output to, optionally gzipped. + description: File to write the output to, potentially gzipped. type: file direction: output example: output.txt diff --git a/src/concat_text/script.sh b/src/concat_text/script.sh index 5efab6f..f25c240 100644 --- a/src/concat_text/script.sh +++ b/src/concat_text/script.sh @@ -1,34 +1,65 @@ #!/usr/bin/env bash -set -euo pipefail +set -eo pipefail -TMPDIR=$(mktemp -d "$meta_temp_dir/concat_text-XXXXXX") -function clean_up { - [[ -d "$TMPDIR" ]] && rm -r "$TMPDIR" +## VIASH START +par_input="README.md;README.qmd" +par_output="concatenated_output.txt" +par_compress_output="true" +## VIASH END + +# --- Function to check for GZIP format using the 'file' command --- +is_gzipped() { + # Ensure the file exists and is not empty before checking + if [ ! -s "$1" ]; then + return 1 + fi + # Get the MIME type of the file. The '-b' option omits the filename from the output. + local mime_type + mime_type=$(file -b --mime-type "$1") + + # Check if the MIME type corresponds to gzip. + # application/gzip is standard, while application/x-gzip is also commonly seen. + if [[ "$mime_type" == "application/gzip" || "$mime_type" == "application/x-gzip" ]]; then + return 0 # 0 indicates success (true in bash) + else + return 1 # 1 indicates failure (false in bash) + fi } -trap clean_up EXIT -par_input="$(echo "$par_input" | tr ';' ' ')" +# Read the ;-separated file paths from the input variable into an array +IFS=";" read -ra input_files <<< "$par_input" -echo -n ">> Check if input is gzipped... " -set +eo pipefail -file $par_input | grep -q 'gzip' -is_zipped="$?" -set -euo pipefail -[[ "$is_zipped" == "0" ]] && echo "yes" || echo "no" +# Process the files if the array contains any paths +if [ ${#input_files[@]} -gt 0 ] && [ -n "${input_files[0]}" ]; then -if [[ "$is_zipped" == "0" ]]; then - echo ">> zcat gzipped files" - zcat $par_input > $TMPDIR/contents + # Ensure the output file is empty before we start + > "$par_output" + + echo "Processing files for -> $par_output" + + # Create a subshell for the loop to group all cat/zcat output. + ( + for file in "${input_files[@]}"; do + if [ -z "$file" ]; then continue; fi # Skip empty entries in the array + + if is_gzipped "$file"; then + zcat "$file" + else + cat "$file" + fi + done + ) | if [ "$par_compress_output" = "true" ]; then + # If compression is enabled, pipe the entire stream to gzip + gzip -c >> "$par_output" + else + # Otherwise, just redirect the stream to the plain text file + cat >> "$par_output" + fi + + echo "Finished creating $par_output." else - echo ">> cat plain files" - cat $par_input > $TMPDIR/contents + echo "No input files provided in \$par_input. Exiting." fi -if [ "$par_gzip_output" == true ]; then - echo ">> Zip output file" - gzip $TMPDIR/contents - mv $TMPDIR/contents.gz $par_output -else - mv $TMPDIR/contents $par_output -fi +echo "Script finished successfully." diff --git a/src/concat_text/test.sh b/src/concat_text/test.sh index 1e75223..2e3900d 100644 --- a/src/concat_text/test.sh +++ b/src/concat_text/test.sh @@ -2,6 +2,65 @@ set -euo pipefail +echo ">> Running concat_text.sh script" + +is_gzipped() { + # Ensure the file exists and is not empty + if [ ! -s "$1" ]; then + return 1 + fi + # Get the MIME type of the file + local mime_type + mime_type=$(file -b --mime-type "$1") + + # Check if the MIME type corresponds to gzip. + if [[ "$mime_type" == "application/gzip" || "$mime_type" == "application/x-gzip" ]]; then + return 0 + else + return 1 + fi +} + +compare_files() { + local file1="$1" + local file2="$2" + if [[ ! -f "$file1" || ! -f "$file2" ]]; then + echo "One of the files does not exist: $file1 or $file2" + return 1 + fi + # decompress file 1 if need be + if is_gzipped "$file1"; then + file1=$(mktemp) + zcat "$1" > "$file1" + fi + # decompress file 2 if need be + if is_gzipped "$file2"; then + file2=$(mktemp) + zcat "$2" > "$file2" + fi + + if cmp -s "$file1" "$file2"; then + echo "Files are identical." + return 0 + else + echo "Files differ." + echo "Found:" + if is_gzipped "$file1"; then + zcat "$file1" | od -c + else + cat "$file1" | od -c + fi + echo "Expected:" + if is_gzipped "$file2"; then + zcat "$file2" | od -c + else + cat "$file2" | od -c + fi + return 1 + fi +} + +## TEST RESOURCES echo ">> Creating test input files file[1-3].txt" INPUT_FILE_1="file1.txt" INPUT_FILE_2="file2.txt" @@ -25,46 +84,31 @@ EOF gzip -k "expected_output.txt" +## RUN TESTS echo ">> Run component on 3 plain input files, plain output" $meta_executable \ --input "$INPUT_FILE_1;$INPUT_FILE_2;$INPUT_FILE_3" \ --output "output1.txt" +compare_files "output1.txt" "expected_output.txt" -[[ ! -f "output1.txt" ]] \ - && echo "Output file output1.txt not found!" && exit 1 -[[ $(cmp "output1.txt" "expected_output.txt") ]] \ - && echo "Output file output1.txt is not as expected!" && exit 1 - -echo ">> Run component on 3 zipped input files, plain output" +echo ">> Run component on mixed input files, plain output" $meta_executable \ - --input "$INPUT_FILE_1.gz;$INPUT_FILE_2.gz;$INPUT_FILE_3.gz" \ + --input "$INPUT_FILE_1.gz;$INPUT_FILE_2;$INPUT_FILE_3.gz" \ --output "output2.txt" - -[[ ! -f "output2.txt" ]] \ - && echo "Output file output2.txt not found!" && exit 1 -[[ $(cmp "output2.txt" "expected_output.txt") ]] \ - && echo "Output file output2.txt is not as expected!" && exit 1 +compare_files "output2.txt" "expected_output.txt" echo ">> Run component on 3 plain input files, zipped output" $meta_executable \ --input "$INPUT_FILE_1;$INPUT_FILE_2;$INPUT_FILE_3" \ --output "output3.txt.gz" \ --gzip_output +compare_files "output3.txt.gz" "expected_output.txt.gz" -[[ ! -f "output3.txt.gz" ]] \ - && echo "Output file output3.txt.gz not found!" && exit 1 -[[ $(cmp "output3.txt.gz" "expected_output.txt.gz") ]] \ - && echo "Output file output3.txt.gz is not as expected!" && exit 1 - -echo ">> Run component on 3 zipped input files, zipped output" +echo ">> Run component on mixed input files, zipped output" $meta_executable \ - --input "$INPUT_FILE_1.gz;$INPUT_FILE_2.gz;$INPUT_FILE_3.gz" \ + --input "$INPUT_FILE_1.gz;$INPUT_FILE_2;$INPUT_FILE_3.gz" \ --output "output4.txt.gz" \ --gzip_output - -[[ ! -f "output4.txt.gz" ]] \ - && echo "Output file output4.txt.gz not found!" && exit 1 -[[ $(cmp "output4.txt.gz" "expected_output.txt.gz") ]] \ - && echo "Output file output4.txt.gz is not as expected!" && exit 1 +compare_files "output4.txt.gz" "expected_output.txt.gz" echo ">> Tests done" diff --git a/src/move_files_to_directory/config.vsh.yaml b/src/move_files_to_directory/config.vsh.yaml index 30c9996..0ac5ebf 100644 --- a/src/move_files_to_directory/config.vsh.yaml +++ b/src/move_files_to_directory/config.vsh.yaml @@ -1,39 +1,52 @@ name: move_files_to_directory -summary: Publish one or multiple files to the same directory -description: This component copies one or multiple files to the same destination directory, creating the output directory if it doesn't exist. +summary: Publish one or multiple files or directories to the same output directory. +description: | + This component copies one or multiple files or directories + to the same destination directory, creating the output directory if it doesn't + exist. authors: - __merge__: /src/_authors/dorien_roosen.yaml roles: [ maintainer ] -arguments: - - name: "--input" - type: file - direction: input - required: true - multiple: true - description: Paths of the files that will be copied into the output directory. - - name: "--output" - type: file - direction: output - required: true - description: Path to output directory - - name: "--keep_symbolic_links" - alternatives: [-d] - type: boolean_true - description: Preserve symbolic links. + +argument_groups: + - name: Inputs + arguments: + - name: "--input" + type: file + direction: input + required: true + multiple: true + description: Paths of the files or directories that will be copied into the output directory. + - name: Outputs + arguments: + - name: "--output" + type: file + direction: output + required: true + description: Path to output directory. + - name: Options + arguments: + - name: "--keep_symbolic_links" + type: boolean_true + description: When set, symbolic links are preserved as symbolic links in the output directory. By default, symbolic links are dereferenced and the target file is copied. + resources: - type: bash_script path: script.sh + test_resources: - type: bash_script path: test.sh engines: -- type: docker - image: debian:latest - setup: + - type: docker + image: debian:latest + setup: - type: apt - packages: [procps] + packages: + - procps + - type: native runners: -- type: executable -- type: nextflow + - type: executable + - type: nextflow diff --git a/src/move_files_to_directory/script.sh b/src/move_files_to_directory/script.sh index 3c5b890..472f1d1 100644 --- a/src/move_files_to_directory/script.sh +++ b/src/move_files_to_directory/script.sh @@ -3,30 +3,37 @@ set -eo pipefail ## VIASH START -par_input="input.txt;input_2.txt" +par_input="input.txt;input_dir.zarr" par_output="output" +par_keep_symbolic_links="false" ## VIASH END - if [[ ! -d "$par_output" ]]; then mkdir -p "$par_output" fi -extra_params=( ) - -if [ "$par_keep_symbolic_links" == "true" ]; then - extra_params+=( "-d" ) +# Set copy flags based on options +if [[ "$par_keep_symbolic_links" == "true" ]]; then + # -a implies -dR --preserve=all + # with -d: same as --no-dereference (and --preserve=links) + # and --no-dereference: never follow symbolic links in SOURCE + # and -R, -r, --recursive: copy directories recursively + # --keep-directory-symlink: if the destination already exists and is a + # symbolic link to a directory, follow the symlink and copy into the directory + # it points to, instead of removing the symlink and creating a real directory in its place. + cp_flags="-a --keep-directory-symlink" +else + # -L: always follow symbolic links in SOURCE + cp_flags="-Lr --preserve=all --no-preserve=link --keep-directory-symlink" fi -# Process multiple input files -IFS=";" read -ra input_files <<< "$par_input" -for file in "${input_files[@]}"; do - # Check if the file exists before copying - if [[ -f "$file" ]]; then - - cp ${extra_params[@]} "$file" "$par_output/" - echo "Copied $file to $par_output/" +# Process multiple input paths (files or directories) +IFS=";" read -ra input_paths <<< "$par_input" +for path in "${input_paths[@]}"; do + if [[ -e "$path" ]] || [[ -L "$path" ]]; then + cp $cp_flags "$path" "$par_output/" + echo "Copied $path to $par_output/" else - echo "Warning: Input file $file does not exist, skipping" + echo "Warning: Input path $path does not exist, skipping" fi done diff --git a/src/move_files_to_directory/test.sh b/src/move_files_to_directory/test.sh index a64ec52..c899ae5 100644 --- a/src/move_files_to_directory/test.sh +++ b/src/move_files_to_directory/test.sh @@ -11,7 +11,7 @@ trap clean_up EXIT touch "$TMPDIR/test_file.txt" touch "$TMPDIR/another_file.txt" -./move_files_to_directory \ +$meta_executable \ --input "$TMPDIR/test_file.txt" \ --input "$TMPDIR/another_file.txt" \ --output "$TMPDIR/test_output" @@ -19,9 +19,23 @@ touch "$TMPDIR/another_file.txt" [[ ! -d "$TMPDIR/test_output" ]] && echo "It seems no output directory is generated" && exit 1 [[ ! -f "$TMPDIR/test_output/test_file.txt" ]] && [[ ! -f test_output/another_file.txt ]] && echo "Output files were not copied to the output directory" && exit 1 +# Test copying a directory +mkdir -p "$TMPDIR/test_dir" +touch "$TMPDIR/test_dir/file_in_dir.txt" +touch "$TMPDIR/test_dir/another_file_in_dir.txt" + +$meta_executable \ + --input "$TMPDIR/test_dir" \ + --output "$TMPDIR/test_output_dir" + +[[ ! -d "$TMPDIR/test_output_dir" ]] && echo "It seems no output directory (test_output_dir) is generated" && exit 1 +[[ ! -d "$TMPDIR/test_output_dir/test_dir" ]] && echo "Input directory was not copied to the output directory" && exit 1 +[[ ! -f "$TMPDIR/test_output_dir/test_dir/file_in_dir.txt" ]] && echo "Files inside the copied directory are missing" && exit 1 +[[ ! -f "$TMPDIR/test_output_dir/test_dir/another_file_in_dir.txt" ]] && echo "Files inside the copied directory are missing" && exit 1 + ln -s "$TMPDIR/test_file.txt" "$TMPDIR/symlink.txt" -./move_files_to_directory \ +$meta_executable \ --input "$TMPDIR/symlink.txt" \ --output "$TMPDIR/test_output_symlink" \ --keep_symbolic_links diff --git a/target/executable/check_disk_space/.config.vsh.yaml b/target/executable/check_disk_space/.config.vsh.yaml index 448ddf6..73a378b 100644 --- a/target/executable/check_disk_space/.config.vsh.yaml +++ b/target/executable/check_disk_space/.config.vsh.yaml @@ -156,8 +156,8 @@ build_info: engine: "docker|native" output: "target/executable/check_disk_space" executable: "target/executable/check_disk_space/check_disk_space" - viash_version: "0.9.4" - git_commit: "32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550" + viash_version: "0.9.7" + git_commit: "b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1" git_remote: "https://github.com/viash-hub/craftbox" package_config: name: "craftbox" @@ -177,7 +177,7 @@ package_config: \ to ensure dependency management and a consistent, reproducible runtime environment.\n\ \ * Unit tested where applicable to ensure components function as expected.\n" info: null - viash_version: "0.9.4" + viash_version: "0.9.7" source: "src" target: "target" config_mods: diff --git a/target/executable/check_disk_space/check_disk_space b/target/executable/check_disk_space/check_disk_space index 97c0160..ec0d69b 100755 --- a/target/executable/check_disk_space/check_disk_space +++ b/target/executable/check_disk_space/check_disk_space @@ -2,7 +2,7 @@ # check_disk_space main # -# This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative +# This wrapper script is auto-generated by viash 0.9.7 and is thus a derivative # work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data # Intuitive. # @@ -446,9 +446,9 @@ function ViashDockerfile { FROM bash:latest ENTRYPOINT [] LABEL org.opencontainers.image.description="Companion container for running component check_disk_space" -LABEL org.opencontainers.image.created="2025-09-01T14:32:38Z" +LABEL org.opencontainers.image.created="2026-06-01T12:52:41Z" LABEL org.opencontainers.image.source="https://github.com/viash-hub/craftbox" -LABEL org.opencontainers.image.revision="32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550" +LABEL org.opencontainers.image.revision="b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1" LABEL org.opencontainers.image.version="main" VIASHDOCKER diff --git a/target/executable/concat_text/.config.vsh.yaml b/target/executable/concat_text/.config.vsh.yaml index f11c9f0..c9fb908 100644 --- a/target/executable/concat_text/.config.vsh.yaml +++ b/target/executable/concat_text/.config.vsh.yaml @@ -26,6 +26,22 @@ authors: - name: "Data Intuitive" href: "https://www.data-intuitive.com" role: "Data Scientist" +- name: "Robrecht Cannoodt" + roles: + - "contributor" + info: + links: + email: "robrecht@data-intuitive.com" + github: "rcannood" + orcid: "0000-0003-3641-729X" + linkedin: "robrechtcannoodt" + organizations: + - name: "Data Intuitive" + href: "https://www.data-intuitive.com" + role: "Data Science Engineer" + - name: "Open Problems" + href: "https://openproblems.bio" + role: "Core Member" argument_groups: - name: "Input arguments" arguments: @@ -34,7 +50,7 @@ argument_groups: description: "A list of (gzipped) text files." info: null example: - - "input?.txt.gz" + - "input.txt.gz" must_exist: true create_parent: true required: true @@ -50,7 +66,7 @@ argument_groups: direction: "input" - type: "file" name: "--output" - description: "File to write the output to, optionally gzipped." + description: "File to write the output to, potentially gzipped." info: null example: - "output.txt" @@ -64,7 +80,7 @@ resources: - type: "bash_script" path: "script.sh" is_executable: true -summary: "Concatenate a number of text files" +summary: "Concatenate multiple (possibly gzipped) text files" description: "Concatenate a number of text files, handle gzipped text files gracefully\ \ and\noptionally gzip the output text file.\n\nThis component is useful for concatening\ \ fastq files from different lanes, for instance.\n" @@ -73,9 +89,8 @@ test_resources: path: "test.sh" is_executable: true info: - improvements: "This component could be improved in 2 ways:\n 1. Allow for a mix\ - \ of zipped and plain input files\n 2. Allow to specify a compression algorithm\ - \ for the output\n" + improvements: "This component could be improved:\n 1. Allow to specify a compression\ + \ algorithm for the output\n" status: "enabled" scope: image: "public" @@ -174,8 +189,8 @@ build_info: engine: "docker|native" output: "target/executable/concat_text" executable: "target/executable/concat_text/concat_text" - viash_version: "0.9.4" - git_commit: "32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550" + viash_version: "0.9.7" + git_commit: "b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1" git_remote: "https://github.com/viash-hub/craftbox" package_config: name: "craftbox" @@ -195,7 +210,7 @@ package_config: \ to ensure dependency management and a consistent, reproducible runtime environment.\n\ \ * Unit tested where applicable to ensure components function as expected.\n" info: null - viash_version: "0.9.4" + viash_version: "0.9.7" source: "src" target: "target" config_mods: diff --git a/target/executable/concat_text/concat_text b/target/executable/concat_text/concat_text index d9cb6f7..2fae04a 100755 --- a/target/executable/concat_text/concat_text +++ b/target/executable/concat_text/concat_text @@ -2,7 +2,7 @@ # concat_text main # -# This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative +# This wrapper script is auto-generated by viash 0.9.7 and is thus a derivative # work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data # Intuitive. # @@ -14,6 +14,7 @@ # Component authors: # * Toni Verbeiren (author, maintainer) # * Dries Schaumont (reviewer) +# * Robrecht Cannoodt (contributor) set -e @@ -451,11 +452,11 @@ FROM alpine:latest ENTRYPOINT [] RUN apk add --no-cache bash procps file -LABEL org.opencontainers.image.authors="Toni Verbeiren, Dries Schaumont" +LABEL org.opencontainers.image.authors="Toni Verbeiren, Dries Schaumont, Robrecht Cannoodt" LABEL org.opencontainers.image.description="Companion container for running component concat_text" -LABEL org.opencontainers.image.created="2025-09-01T14:32:38Z" +LABEL org.opencontainers.image.created="2026-06-01T12:52:41Z" LABEL org.opencontainers.image.source="https://github.com/viash-hub/craftbox" -LABEL org.opencontainers.image.revision="32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550" +LABEL org.opencontainers.image.revision="b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -584,7 +585,7 @@ function ViashHelp { echo "Input arguments:" echo " --input" echo " type: file, required parameter, multiple values allowed, file must exist" - echo " example: input?.txt.gz" + echo " example: input.txt.gz" echo " A list of (gzipped) text files." echo "" echo "Output arguments:" @@ -595,7 +596,7 @@ function ViashHelp { echo " --output" echo " type: file, output, file must exist" echo " example: output.txt" - echo " File to write the output to, optionally gzipped." + echo " File to write the output to, potentially gzipped." echo "" echo "Viash built in Computational Requirements:" echo " ---cpus=INT" @@ -1080,6 +1081,10 @@ function interrupt { trap clean_up EXIT trap interrupt INT SIGINT cat > "\$tempscript" << 'VIASHMAIN' +#!/usr/bin/env bash + +set -eo pipefail + ## VIASH START # The following code has been auto-generated by Viash. $( if [ ! -z ${VIASH_PAR_INPUT+x} ]; then echo "${VIASH_PAR_INPUT}" | sed "s#'#'\"'\"'#g;s#.*#par_input='&'#" ; else echo "# par_input="; fi ) @@ -1105,40 +1110,62 @@ $( if [ ! -z ${VIASH_META_MEMORY_TIB+x} ]; then echo "${VIASH_META_MEMORY_TIB}" $( if [ ! -z ${VIASH_META_MEMORY_PIB+x} ]; then echo "${VIASH_META_MEMORY_PIB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_pib='&'#" ; else echo "# meta_memory_pib="; fi ) ## VIASH END -#!/usr/bin/env bash -set -euo pipefail - -TMPDIR=\$(mktemp -d "\$meta_temp_dir/concat_text-XXXXXX") -function clean_up { - [[ -d "\$TMPDIR" ]] && rm -r "\$TMPDIR" +# --- Function to check for GZIP format using the 'file' command --- +is_gzipped() { + # Ensure the file exists and is not empty before checking + if [ ! -s "\$1" ]; then + return 1 + fi + # Get the MIME type of the file. The '-b' option omits the filename from the output. + local mime_type + mime_type=\$(file -b --mime-type "\$1") + + # Check if the MIME type corresponds to gzip. + # application/gzip is standard, while application/x-gzip is also commonly seen. + if [[ "\$mime_type" == "application/gzip" || "\$mime_type" == "application/x-gzip" ]]; then + return 0 # 0 indicates success (true in bash) + else + return 1 # 1 indicates failure (false in bash) + fi } -trap clean_up EXIT -par_input="\$(echo "\$par_input" | tr ';' ' ')" +# Read the ;-separated file paths from the input variable into an array +IFS=";" read -ra input_files <<< "\$par_input" -echo -n ">> Check if input is gzipped... " -set +eo pipefail -file \$par_input | grep -q 'gzip' -is_zipped="\$?" -set -euo pipefail -[[ "\$is_zipped" == "0" ]] && echo "yes" || echo "no" +# Process the files if the array contains any paths +if [ \${#input_files[@]} -gt 0 ] && [ -n "\${input_files[0]}" ]; then -if [[ "\$is_zipped" == "0" ]]; then - echo ">> zcat gzipped files" - zcat \$par_input > \$TMPDIR/contents + # Ensure the output file is empty before we start + > "\$par_output" + + echo "Processing files for -> \$par_output" + + # Create a subshell for the loop to group all cat/zcat output. + ( + for file in "\${input_files[@]}"; do + if [ -z "\$file" ]; then continue; fi # Skip empty entries in the array + + if is_gzipped "\$file"; then + zcat "\$file" + else + cat "\$file" + fi + done + ) | if [ "\$par_compress_output" = "true" ]; then + # If compression is enabled, pipe the entire stream to gzip + gzip -c >> "\$par_output" + else + # Otherwise, just redirect the stream to the plain text file + cat >> "\$par_output" + fi + + echo "Finished creating \$par_output." else - echo ">> cat plain files" - cat \$par_input > \$TMPDIR/contents + echo "No input files provided in \\\$par_input. Exiting." fi -if [ "\$par_gzip_output" == true ]; then - echo ">> Zip output file" - gzip \$TMPDIR/contents - mv \$TMPDIR/contents.gz \$par_output -else - mv \$TMPDIR/contents \$par_output -fi +echo "Script finished successfully." VIASHMAIN bash "\$tempscript" & wait "\$!" diff --git a/target/executable/csv2fasta/.config.vsh.yaml b/target/executable/csv2fasta/.config.vsh.yaml index 936d447..cb1b606 100644 --- a/target/executable/csv2fasta/.config.vsh.yaml +++ b/target/executable/csv2fasta/.config.vsh.yaml @@ -256,8 +256,8 @@ build_info: engine: "docker|native" output: "target/executable/csv2fasta" executable: "target/executable/csv2fasta/csv2fasta" - viash_version: "0.9.4" - git_commit: "32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550" + viash_version: "0.9.7" + git_commit: "b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1" git_remote: "https://github.com/viash-hub/craftbox" package_config: name: "craftbox" @@ -277,7 +277,7 @@ package_config: \ to ensure dependency management and a consistent, reproducible runtime environment.\n\ \ * Unit tested where applicable to ensure components function as expected.\n" info: null - viash_version: "0.9.4" + viash_version: "0.9.7" source: "src" target: "target" config_mods: diff --git a/target/executable/csv2fasta/csv2fasta b/target/executable/csv2fasta/csv2fasta index 5244a03..d486816 100755 --- a/target/executable/csv2fasta/csv2fasta +++ b/target/executable/csv2fasta/csv2fasta @@ -2,7 +2,7 @@ # csv2fasta main # -# This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative +# This wrapper script is auto-generated by viash 0.9.7 and is thus a derivative # work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data # Intuitive. # @@ -458,9 +458,9 @@ RUN pip install --upgrade pip && \ LABEL org.opencontainers.image.authors="Dries Schaumont, Robrecht Cannoodt" LABEL org.opencontainers.image.description="Companion container for running component csv2fasta" -LABEL org.opencontainers.image.created="2025-09-01T14:32:38Z" +LABEL org.opencontainers.image.created="2026-06-01T12:52:41Z" LABEL org.opencontainers.image.source="https://github.com/viash-hub/craftbox" -LABEL org.opencontainers.image.revision="32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550" +LABEL org.opencontainers.image.revision="b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1" LABEL org.opencontainers.image.version="main" VIASHDOCKER diff --git a/target/executable/move_files_to_directory/.config.vsh.yaml b/target/executable/move_files_to_directory/.config.vsh.yaml index 351d2fb..24171aa 100644 --- a/target/executable/move_files_to_directory/.config.vsh.yaml +++ b/target/executable/move_files_to_directory/.config.vsh.yaml @@ -14,11 +14,12 @@ authors: href: "https://www.data-intuitive.com" role: "Data Scientist" argument_groups: -- name: "Arguments" +- name: "Inputs" arguments: - type: "file" name: "--input" - description: "Paths of the files that will be copied into the output directory." + description: "Paths of the files or directories that will be copied into the output\ + \ directory." info: null must_exist: true create_parent: true @@ -26,9 +27,11 @@ argument_groups: direction: "input" multiple: true multiple_sep: ";" +- name: "Outputs" + arguments: - type: "file" name: "--output" - description: "Path to output directory" + description: "Path to output directory." info: null must_exist: true create_parent: true @@ -36,20 +39,22 @@ argument_groups: direction: "output" multiple: false multiple_sep: ";" +- name: "Options" + arguments: - type: "boolean_true" name: "--keep_symbolic_links" - alternatives: - - "-d" - description: "Preserve symbolic links." + description: "When set, symbolic links are preserved as symbolic links in the\ + \ output directory. By default, symbolic links are dereferenced and the target\ + \ file is copied." info: null direction: "input" resources: - type: "bash_script" path: "script.sh" is_executable: true -summary: "Publish one or multiple files to the same directory" -description: "This component copies one or multiple files to the same destination\ - \ directory, creating the output directory if it doesn't exist." +summary: "Publish one or multiple files or directories to the same output directory." +description: "This component copies one or multiple files or directories\nto the same\ + \ destination directory, creating the output directory if it doesn't\nexist.\n" test_resources: - type: "bash_script" path: "test.sh" @@ -146,14 +151,16 @@ engines: cmd: null - type: "native" id: "native" +- type: "native" + id: "native" build_info: config: "src/move_files_to_directory/config.vsh.yaml" runner: "executable" - engine: "docker|native" + engine: "docker|native|native" output: "target/executable/move_files_to_directory" executable: "target/executable/move_files_to_directory/move_files_to_directory" - viash_version: "0.9.4" - git_commit: "32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550" + viash_version: "0.9.7" + git_commit: "b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1" git_remote: "https://github.com/viash-hub/craftbox" package_config: name: "craftbox" @@ -173,7 +180,7 @@ package_config: \ to ensure dependency management and a consistent, reproducible runtime environment.\n\ \ * Unit tested where applicable to ensure components function as expected.\n" info: null - viash_version: "0.9.4" + viash_version: "0.9.7" source: "src" target: "target" config_mods: diff --git a/target/executable/move_files_to_directory/move_files_to_directory b/target/executable/move_files_to_directory/move_files_to_directory index 87e1f4d..6253431 100755 --- a/target/executable/move_files_to_directory/move_files_to_directory +++ b/target/executable/move_files_to_directory/move_files_to_directory @@ -2,7 +2,7 @@ # move_files_to_directory main # -# This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative +# This wrapper script is auto-generated by viash 0.9.7 and is thus a derivative # work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data # Intuitive. # @@ -454,9 +454,9 @@ RUN apt-get update && \ LABEL org.opencontainers.image.authors="Dorien Roosen" LABEL org.opencontainers.image.description="Companion container for running component move_files_to_directory" -LABEL org.opencontainers.image.created="2025-09-01T14:32:38Z" +LABEL org.opencontainers.image.created="2026-06-01T12:52:41Z" LABEL org.opencontainers.image.source="https://github.com/viash-hub/craftbox" -LABEL org.opencontainers.image.revision="32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550" +LABEL org.opencontainers.image.revision="b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1" LABEL org.opencontainers.image.version="main" VIASHDOCKER @@ -576,21 +576,27 @@ VIASH_DOCKER_RUN_ARGS=(-i --rm) function ViashHelp { echo "move_files_to_directory main" echo "" - echo "This component copies one or multiple files to the same destination directory," - echo "creating the output directory if it doesn't exist." + echo "This component copies one or multiple files or directories" + echo "to the same destination directory, creating the output directory if it doesn't" + echo "exist." echo "" - echo "Arguments:" + echo "Inputs:" echo " --input" echo " type: file, required parameter, multiple values allowed, file must exist" - echo " Paths of the files that will be copied into the output directory." + echo " Paths of the files or directories that will be copied into the output" + echo " directory." echo "" + echo "Outputs:" echo " --output" echo " type: file, required parameter, output, file must exist" - echo " Path to output directory" + echo " Path to output directory." echo "" - echo " -d, --keep_symbolic_links" + echo "Options:" + echo " --keep_symbolic_links" echo " type: boolean_true" - echo " Preserve symbolic links." + echo " When set, symbolic links are preserved as symbolic links in the output" + echo " directory. By default, symbolic links are dereferenced and the target" + echo " file is copied." echo "" echo "Viash built in Computational Requirements:" echo " ---cpus=INT" @@ -613,7 +619,7 @@ function ViashHelp { echo "" echo "Viash built in Engines:" echo " ---engine=ENGINE_ID" - echo " Specify the engine to use. Options are: docker, native." + echo " Specify the engine to use. Options are: docker, native, native." echo " Default: docker" } @@ -675,11 +681,6 @@ while [[ $# -gt 0 ]]; do VIASH_PAR_KEEP_SYMBOLIC_LINKS=true shift 1 ;; - -d) - [ -n "$VIASH_PAR_KEEP_SYMBOLIC_LINKS" ] && ViashError Bad arguments for option \'-d\': \'$VIASH_PAR_KEEP_SYMBOLIC_LINKS\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_KEEP_SYMBOLIC_LINKS=true - shift 1 - ;; ---engine) VIASH_ENGINE_ID="$2" shift 2 @@ -753,12 +754,12 @@ done eval set -- $VIASH_POSITIONAL_ARGS -if [ "$VIASH_ENGINE_ID" == "native" ] ; then +if [ "$VIASH_ENGINE_ID" == "native" ] || [ "$VIASH_ENGINE_ID" == "native" ] ; then VIASH_ENGINE_TYPE='native' elif [ "$VIASH_ENGINE_ID" == "docker" ] ; then VIASH_ENGINE_TYPE='docker' else - ViashError "Engine '$VIASH_ENGINE_ID' is not recognized. Options are: docker, native." + ViashError "Engine '$VIASH_ENGINE_ID' is not recognized. Options are: docker, native, native." exit 1 fi @@ -989,7 +990,7 @@ if [ ! -z "$VIASH_PAR_OUTPUT" ] && [ ! -d "$(dirname "$VIASH_PAR_OUTPUT")" ]; th mkdir -p "$(dirname "$VIASH_PAR_OUTPUT")" fi -if [ "$VIASH_ENGINE_ID" == "native" ] ; then +if [ "$VIASH_ENGINE_ID" == "native" ] || [ "$VIASH_ENGINE_ID" == "native" ] ; then if [ "$VIASH_MODE" == "run" ]; then VIASH_CMD="bash" else @@ -1114,27 +1115,33 @@ $( if [ ! -z ${VIASH_META_MEMORY_PIB+x} ]; then echo "${VIASH_META_MEMORY_PIB}" ## VIASH END - if [[ ! -d "\$par_output" ]]; then mkdir -p "\$par_output" fi -extra_params=( ) - -if [ "\$par_keep_symbolic_links" == "true" ]; then - extra_params+=( "-d" ) +# Set copy flags based on options +if [[ "\$par_keep_symbolic_links" == "true" ]]; then + # -a implies -dR --preserve=all + # with -d: same as --no-dereference (and --preserve=links) + # and --no-dereference: never follow symbolic links in SOURCE + # and -R, -r, --recursive: copy directories recursively + # --keep-directory-symlink: if the destination already exists and is a + # symbolic link to a directory, follow the symlink and copy into the directory + # it points to, instead of removing the symlink and creating a real directory in its place. + cp_flags="-a --keep-directory-symlink" +else + # -L: always follow symbolic links in SOURCE + cp_flags="-Lr --preserve=all --no-preserve=link --keep-directory-symlink" fi -# Process multiple input files -IFS=";" read -ra input_files <<< "\$par_input" -for file in "\${input_files[@]}"; do - # Check if the file exists before copying - if [[ -f "\$file" ]]; then - - cp \${extra_params[@]} "\$file" "\$par_output/" - echo "Copied \$file to \$par_output/" +# Process multiple input paths (files or directories) +IFS=";" read -ra input_paths <<< "\$par_input" +for path in "\${input_paths[@]}"; do + if [[ -e "\$path" ]] || [[ -L "\$path" ]]; then + cp \$cp_flags "\$path" "\$par_output/" + echo "Copied \$path to \$par_output/" else - echo "Warning: Input file \$file does not exist, skipping" + echo "Warning: Input path \$path does not exist, skipping" fi done VIASHMAIN diff --git a/target/executable/sync_resources/.config.vsh.yaml b/target/executable/sync_resources/.config.vsh.yaml index 74fd1b7..69b6cc5 100644 --- a/target/executable/sync_resources/.config.vsh.yaml +++ b/target/executable/sync_resources/.config.vsh.yaml @@ -196,8 +196,8 @@ build_info: engine: "docker|native" output: "target/executable/sync_resources" executable: "target/executable/sync_resources/sync_resources" - viash_version: "0.9.4" - git_commit: "32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550" + viash_version: "0.9.7" + git_commit: "b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1" git_remote: "https://github.com/viash-hub/craftbox" package_config: name: "craftbox" @@ -217,7 +217,7 @@ package_config: \ to ensure dependency management and a consistent, reproducible runtime environment.\n\ \ * Unit tested where applicable to ensure components function as expected.\n" info: null - viash_version: "0.9.4" + viash_version: "0.9.7" source: "src" target: "target" config_mods: diff --git a/target/executable/sync_resources/sync_resources b/target/executable/sync_resources/sync_resources index 2fe5a68..37afde8 100755 --- a/target/executable/sync_resources/sync_resources +++ b/target/executable/sync_resources/sync_resources @@ -2,7 +2,7 @@ # sync_resources main # -# This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative +# This wrapper script is auto-generated by viash 0.9.7 and is thus a derivative # work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data # Intuitive. # @@ -455,9 +455,9 @@ RUN rclone config create s3 s3 anonymous=true RUN rclone config create gs gcs anonymous=true LABEL org.opencontainers.image.authors="Robrecht Cannoodt, Dries Schaumont" LABEL org.opencontainers.image.description="Companion container for running component sync_resources" -LABEL org.opencontainers.image.created="2025-09-01T14:32:37Z" +LABEL org.opencontainers.image.created="2026-06-01T12:52:41Z" LABEL org.opencontainers.image.source="https://github.com/viash-hub/craftbox" -LABEL org.opencontainers.image.revision="32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550" +LABEL org.opencontainers.image.revision="b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1" LABEL org.opencontainers.image.version="main" VIASHDOCKER diff --git a/target/executable/untar/.config.vsh.yaml b/target/executable/untar/.config.vsh.yaml index 19f3c99..e8579fa 100644 --- a/target/executable/untar/.config.vsh.yaml +++ b/target/executable/untar/.config.vsh.yaml @@ -181,8 +181,8 @@ build_info: engine: "docker|native" output: "target/executable/untar" executable: "target/executable/untar/untar" - viash_version: "0.9.4" - git_commit: "32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550" + viash_version: "0.9.7" + git_commit: "b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1" git_remote: "https://github.com/viash-hub/craftbox" package_config: name: "craftbox" @@ -202,7 +202,7 @@ package_config: \ to ensure dependency management and a consistent, reproducible runtime environment.\n\ \ * Unit tested where applicable to ensure components function as expected.\n" info: null - viash_version: "0.9.4" + viash_version: "0.9.7" source: "src" target: "target" config_mods: diff --git a/target/executable/untar/untar b/target/executable/untar/untar index 2610b90..f430be7 100755 --- a/target/executable/untar/untar +++ b/target/executable/untar/untar @@ -2,7 +2,7 @@ # untar main # -# This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative +# This wrapper script is auto-generated by viash 0.9.7 and is thus a derivative # work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data # Intuitive. # @@ -455,9 +455,9 @@ RUN apt-get update && \ LABEL org.opencontainers.image.authors="Dries Schaumont, Robrecht Cannoodt" LABEL org.opencontainers.image.description="Companion container for running component untar" -LABEL org.opencontainers.image.created="2025-09-01T14:32:37Z" +LABEL org.opencontainers.image.created="2026-06-01T12:52:41Z" LABEL org.opencontainers.image.source="https://github.com/viash-hub/craftbox" -LABEL org.opencontainers.image.revision="32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550" +LABEL org.opencontainers.image.revision="b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1" LABEL org.opencontainers.image.version="main" VIASHDOCKER diff --git a/target/nextflow/check_disk_space/.config.vsh.yaml b/target/nextflow/check_disk_space/.config.vsh.yaml index 5d62189..b41ca9f 100644 --- a/target/nextflow/check_disk_space/.config.vsh.yaml +++ b/target/nextflow/check_disk_space/.config.vsh.yaml @@ -156,8 +156,8 @@ build_info: engine: "docker|native" output: "target/nextflow/check_disk_space" executable: "target/nextflow/check_disk_space/main.nf" - viash_version: "0.9.4" - git_commit: "32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550" + viash_version: "0.9.7" + git_commit: "b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1" git_remote: "https://github.com/viash-hub/craftbox" package_config: name: "craftbox" @@ -177,7 +177,7 @@ package_config: \ to ensure dependency management and a consistent, reproducible runtime environment.\n\ \ * Unit tested where applicable to ensure components function as expected.\n" info: null - viash_version: "0.9.4" + viash_version: "0.9.7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/check_disk_space/main.nf b/target/nextflow/check_disk_space/main.nf index fba7643..0979828 100644 --- a/target/nextflow/check_disk_space/main.nf +++ b/target/nextflow/check_disk_space/main.nf @@ -1,6 +1,6 @@ // check_disk_space main // -// This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative +// This wrapper script is auto-generated by viash 0.9.7 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data // Intuitive. // @@ -1350,47 +1350,42 @@ def readCsv(file_path) { def splitRegex = java.util.regex.Pattern.compile(''',(?=(?:[^"]*"[^"]*")*[^"]*$)''') def removeQuote = java.util.regex.Pattern.compile('''"(.*)"''') - def br = java.nio.file.Files.newBufferedReader(inputFile) + java.nio.file.Files.newBufferedReader(inputFile).withCloseable { br -> + def row = 0 + def header = null + def line - def row = -1 - def header = null - while (br.ready() && header == null) { - def line = br.readLine() - row++ - if (!line.startsWith("#")) { - header = splitRegex.split(line, -1).collect{field -> - m = removeQuote.matcher(field) - m.find() ? m.replaceFirst('$1') : field - } - } - } - assert header != null: "CSV file should contain a header" - - while (br.ready()) { - def line = br.readLine() - row++ - if (line == null) { - br.close() - break - } - - if (!line.startsWith("#")) { - def predata = splitRegex.split(line, -1) - def data = predata.collect{field -> - if (field == "") { - return null - } - def m = removeQuote.matcher(field) - if (m.find()) { - return m.replaceFirst('$1') - } else { - return field + while (header == null && (line = br.readLine()) != null) { + if (!line.startsWith("#")) { + header = splitRegex.split(line, -1).collect { field -> + def m = removeQuote.matcher(field) + m.find() ? m.replaceFirst('$1') : field } } - assert header.size() == data.size(): "Row $row should contain the same number as fields as the header" - - def dataMap = [header, data].transpose().collectEntries().findAll{it.value != null} - output.add(dataMap) + row++ + } + assert header != null : "CSV file should contain a header" + + while ((line = br.readLine()) != null) { + row++ + if (!line.startsWith("#")) { + def predata = splitRegex.split(line, -1) + def data = predata.collect { field -> + if (field == "") { + return null + } + def m = removeQuote.matcher(field) + if (m.find()) { + return m.replaceFirst('$1') + } else { + return field + } + } + assert header.size() == data.size() : "Row $row should contain the same number as fields as the header" + + def dataMap = [header, data].transpose().collectEntries().findAll { it.value != null } + output.add(dataMap) + } } } @@ -1706,10 +1701,25 @@ process publishFilesProc { ] .transpose() .collectMany{infile, outfile -> - if (infile.toString() != outfile.toString()) { + def infileString = infile.toString() + def outfileString = outfile.toString() + if (infileString != outfileString) { + /* Trailing slashes are removed from both the source and destination arguments. + From source arguments, this is useful when a source argument may have a trailing slash + and specify a symbolic link to a directory. Without removing the slash, cp will dereference + the symbolic link. + See https://www.gnu.org/software/coreutils/manual/html_node/Trailing-slashes.html#Trailing-slashes-1 + + For the destination path addding a trailing slash is a problem when publishing directories: + it requires the destination directory to exist. This fails because we only create the parent + directories first. + */ + def regexTrailingSlashes = ~/\/+$/ + def infileNoTrailingSlash = infileString - regexTrailingSlashes + def outfileNoTrailingSlash = outfileString - regexTrailingSlashes [ - "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"", - "cp -r '${infile.toString()}' '${outfile.toString()}'" + "[ -d \"\$(dirname '${outfileNoTrailingSlash}')\" ] || mkdir -p \"\$(dirname '${outfileNoTrailingSlash}')\"", + "cp -a '${infileNoTrailingSlash}' '${outfileNoTrailingSlash}'" ] } else { // no need to copy if infile is the same as outfile @@ -3215,8 +3225,8 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/check_disk_space", - "viash_version" : "0.9.4", - "git_commit" : "32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550", + "viash_version" : "0.9.7", + "git_commit" : "b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1", "git_remote" : "https://github.com/viash-hub/craftbox" }, "package_config" : { @@ -3224,7 +3234,7 @@ meta = [ "version" : "main", "summary" : "A collection of custom-tailored scripts and applied utilities built with Viash.\n", "description" : "`craftbox` is a curated collection of custom scripts and utilities designed to tackle context-specific tasks.\n\nEmphasizing the Viash principles, `craftbox` components aim for **reusability**, **reproducibility**, and adherence to **best practices**. Key features generally include:\n\n* **Standalone & Nextflow Ready:** Components are built to run directly via the command line or be smoothly integrated into Nextflow workflows.\n* **Custom Implementations:** Contains scripts and tools developed for particular tasks that may not be found in broader collections.\n* **High Quality Standards (promoted by Viash):**\n * Clear documentation for components and their parameters.\n * Full exposure of underlying script/tool arguments for fine-grained control.\n * Containerized (Docker) to ensure dependency management and a consistent, reproducible runtime environment.\n * Unit tested where applicable to ensure components function as expected.\n", - "viash_version" : "0.9.4", + "viash_version" : "0.9.7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/concat_text/.config.vsh.yaml b/target/nextflow/concat_text/.config.vsh.yaml index 36741cb..70758ea 100644 --- a/target/nextflow/concat_text/.config.vsh.yaml +++ b/target/nextflow/concat_text/.config.vsh.yaml @@ -26,6 +26,22 @@ authors: - name: "Data Intuitive" href: "https://www.data-intuitive.com" role: "Data Scientist" +- name: "Robrecht Cannoodt" + roles: + - "contributor" + info: + links: + email: "robrecht@data-intuitive.com" + github: "rcannood" + orcid: "0000-0003-3641-729X" + linkedin: "robrechtcannoodt" + organizations: + - name: "Data Intuitive" + href: "https://www.data-intuitive.com" + role: "Data Science Engineer" + - name: "Open Problems" + href: "https://openproblems.bio" + role: "Core Member" argument_groups: - name: "Input arguments" arguments: @@ -34,7 +50,7 @@ argument_groups: description: "A list of (gzipped) text files." info: null example: - - "input?.txt.gz" + - "input.txt.gz" must_exist: true create_parent: true required: true @@ -50,7 +66,7 @@ argument_groups: direction: "input" - type: "file" name: "--output" - description: "File to write the output to, optionally gzipped." + description: "File to write the output to, potentially gzipped." info: null example: - "output.txt" @@ -64,7 +80,7 @@ resources: - type: "bash_script" path: "script.sh" is_executable: true -summary: "Concatenate a number of text files" +summary: "Concatenate multiple (possibly gzipped) text files" description: "Concatenate a number of text files, handle gzipped text files gracefully\ \ and\noptionally gzip the output text file.\n\nThis component is useful for concatening\ \ fastq files from different lanes, for instance.\n" @@ -73,9 +89,8 @@ test_resources: path: "test.sh" is_executable: true info: - improvements: "This component could be improved in 2 ways:\n 1. Allow for a mix\ - \ of zipped and plain input files\n 2. Allow to specify a compression algorithm\ - \ for the output\n" + improvements: "This component could be improved:\n 1. Allow to specify a compression\ + \ algorithm for the output\n" status: "enabled" scope: image: "public" @@ -174,8 +189,8 @@ build_info: engine: "docker|native" output: "target/nextflow/concat_text" executable: "target/nextflow/concat_text/main.nf" - viash_version: "0.9.4" - git_commit: "32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550" + viash_version: "0.9.7" + git_commit: "b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1" git_remote: "https://github.com/viash-hub/craftbox" package_config: name: "craftbox" @@ -195,7 +210,7 @@ package_config: \ to ensure dependency management and a consistent, reproducible runtime environment.\n\ \ * Unit tested where applicable to ensure components function as expected.\n" info: null - viash_version: "0.9.4" + viash_version: "0.9.7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/concat_text/main.nf b/target/nextflow/concat_text/main.nf index 8c88ed2..2f82fcf 100644 --- a/target/nextflow/concat_text/main.nf +++ b/target/nextflow/concat_text/main.nf @@ -1,6 +1,6 @@ // concat_text main // -// This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative +// This wrapper script is auto-generated by viash 0.9.7 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data // Intuitive. // @@ -12,6 +12,7 @@ // Component authors: // * Toni Verbeiren (author, maintainer) // * Dries Schaumont (reviewer) +// * Robrecht Cannoodt (contributor) //////////////////////////// // VDSL3 helper functions // @@ -1354,47 +1355,42 @@ def readCsv(file_path) { def splitRegex = java.util.regex.Pattern.compile(''',(?=(?:[^"]*"[^"]*")*[^"]*$)''') def removeQuote = java.util.regex.Pattern.compile('''"(.*)"''') - def br = java.nio.file.Files.newBufferedReader(inputFile) + java.nio.file.Files.newBufferedReader(inputFile).withCloseable { br -> + def row = 0 + def header = null + def line - def row = -1 - def header = null - while (br.ready() && header == null) { - def line = br.readLine() - row++ - if (!line.startsWith("#")) { - header = splitRegex.split(line, -1).collect{field -> - m = removeQuote.matcher(field) - m.find() ? m.replaceFirst('$1') : field - } - } - } - assert header != null: "CSV file should contain a header" - - while (br.ready()) { - def line = br.readLine() - row++ - if (line == null) { - br.close() - break - } - - if (!line.startsWith("#")) { - def predata = splitRegex.split(line, -1) - def data = predata.collect{field -> - if (field == "") { - return null - } - def m = removeQuote.matcher(field) - if (m.find()) { - return m.replaceFirst('$1') - } else { - return field + while (header == null && (line = br.readLine()) != null) { + if (!line.startsWith("#")) { + header = splitRegex.split(line, -1).collect { field -> + def m = removeQuote.matcher(field) + m.find() ? m.replaceFirst('$1') : field } } - assert header.size() == data.size(): "Row $row should contain the same number as fields as the header" - - def dataMap = [header, data].transpose().collectEntries().findAll{it.value != null} - output.add(dataMap) + row++ + } + assert header != null : "CSV file should contain a header" + + while ((line = br.readLine()) != null) { + row++ + if (!line.startsWith("#")) { + def predata = splitRegex.split(line, -1) + def data = predata.collect { field -> + if (field == "") { + return null + } + def m = removeQuote.matcher(field) + if (m.find()) { + return m.replaceFirst('$1') + } else { + return field + } + } + assert header.size() == data.size() : "Row $row should contain the same number as fields as the header" + + def dataMap = [header, data].transpose().collectEntries().findAll { it.value != null } + output.add(dataMap) + } } } @@ -1710,10 +1706,25 @@ process publishFilesProc { ] .transpose() .collectMany{infile, outfile -> - if (infile.toString() != outfile.toString()) { + def infileString = infile.toString() + def outfileString = outfile.toString() + if (infileString != outfileString) { + /* Trailing slashes are removed from both the source and destination arguments. + From source arguments, this is useful when a source argument may have a trailing slash + and specify a symbolic link to a directory. Without removing the slash, cp will dereference + the symbolic link. + See https://www.gnu.org/software/coreutils/manual/html_node/Trailing-slashes.html#Trailing-slashes-1 + + For the destination path addding a trailing slash is a problem when publishing directories: + it requires the destination directory to exist. This fails because we only create the parent + directories first. + */ + def regexTrailingSlashes = ~/\/+$/ + def infileNoTrailingSlash = infileString - regexTrailingSlashes + def outfileNoTrailingSlash = outfileString - regexTrailingSlashes [ - "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"", - "cp -r '${infile.toString()}' '${outfile.toString()}'" + "[ -d \"\$(dirname '${outfileNoTrailingSlash}')\" ] || mkdir -p \"\$(dirname '${outfileNoTrailingSlash}')\"", + "cp -a '${infileNoTrailingSlash}' '${outfileNoTrailingSlash}'" ] } else { // no need to copy if infile is the same as outfile @@ -3077,6 +3088,32 @@ meta = [ } ] } + }, + { + "name" : "Robrecht Cannoodt", + "roles" : [ + "contributor" + ], + "info" : { + "links" : { + "email" : "robrecht@data-intuitive.com", + "github" : "rcannood", + "orcid" : "0000-0003-3641-729X", + "linkedin" : "robrechtcannoodt" + }, + "organizations" : [ + { + "name" : "Data Intuitive", + "href" : "https://www.data-intuitive.com", + "role" : "Data Science Engineer" + }, + { + "name" : "Open Problems", + "href" : "https://openproblems.bio", + "role" : "Core Member" + } + ] + } } ], "argument_groups" : [ @@ -3088,7 +3125,7 @@ meta = [ "name" : "--input", "description" : "A list of (gzipped) text files.", "example" : [ - "input?.txt.gz" + "input.txt.gz" ], "must_exist" : true, "create_parent" : true, @@ -3111,7 +3148,7 @@ meta = [ { "type" : "file", "name" : "--output", - "description" : "File to write the output to, optionally gzipped.", + "description" : "File to write the output to, potentially gzipped.", "example" : [ "output.txt" ], @@ -3132,7 +3169,7 @@ meta = [ "is_executable" : true } ], - "summary" : "Concatenate a number of text files", + "summary" : "Concatenate multiple (possibly gzipped) text files", "description" : "Concatenate a number of text files, handle gzipped text files gracefully and\noptionally gzip the output text file.\n\nThis component is useful for concatening fastq files from different lanes, for instance.\n", "test_resources" : [ { @@ -3142,7 +3179,7 @@ meta = [ } ], "info" : { - "improvements" : "This component could be improved in 2 ways:\n 1. Allow for a mix of zipped and plain input files\n 2. Allow to specify a compression algorithm for the output\n" + "improvements" : "This component could be improved:\n 1. Allow to specify a compression algorithm for the output\n" }, "status" : "enabled", "scope" : { @@ -3261,8 +3298,8 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/concat_text", - "viash_version" : "0.9.4", - "git_commit" : "32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550", + "viash_version" : "0.9.7", + "git_commit" : "b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1", "git_remote" : "https://github.com/viash-hub/craftbox" }, "package_config" : { @@ -3270,7 +3307,7 @@ meta = [ "version" : "main", "summary" : "A collection of custom-tailored scripts and applied utilities built with Viash.\n", "description" : "`craftbox` is a curated collection of custom scripts and utilities designed to tackle context-specific tasks.\n\nEmphasizing the Viash principles, `craftbox` components aim for **reusability**, **reproducibility**, and adherence to **best practices**. Key features generally include:\n\n* **Standalone & Nextflow Ready:** Components are built to run directly via the command line or be smoothly integrated into Nextflow workflows.\n* **Custom Implementations:** Contains scripts and tools developed for particular tasks that may not be found in broader collections.\n* **High Quality Standards (promoted by Viash):**\n * Clear documentation for components and their parameters.\n * Full exposure of underlying script/tool arguments for fine-grained control.\n * Containerized (Docker) to ensure dependency management and a consistent, reproducible runtime environment.\n * Unit tested where applicable to ensure components function as expected.\n", - "viash_version" : "0.9.4", + "viash_version" : "0.9.7", "source" : "src", "target" : "target", "config_mods" : [ @@ -3304,6 +3341,10 @@ def innerWorkflowFactory(args) { def rawScript = '''set -e tempscript=".viash_script.sh" cat > "$tempscript" << VIASHMAIN +#!/usr/bin/env bash + +set -eo pipefail + ## VIASH START # The following code has been auto-generated by Viash. $( if [ ! -z ${VIASH_PAR_INPUT+x} ]; then echo "${VIASH_PAR_INPUT}" | sed "s#'#'\\"'\\"'#g;s#.*#par_input='&'#" ; else echo "# par_input="; fi ) @@ -3329,40 +3370,62 @@ $( if [ ! -z ${VIASH_META_MEMORY_TIB+x} ]; then echo "${VIASH_META_MEMORY_TIB}" $( if [ ! -z ${VIASH_META_MEMORY_PIB+x} ]; then echo "${VIASH_META_MEMORY_PIB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_pib='&'#" ; else echo "# meta_memory_pib="; fi ) ## VIASH END -#!/usr/bin/env bash -set -euo pipefail - -TMPDIR=\\$(mktemp -d "\\$meta_temp_dir/concat_text-XXXXXX") -function clean_up { - [[ -d "\\$TMPDIR" ]] && rm -r "\\$TMPDIR" +# --- Function to check for GZIP format using the 'file' command --- +is_gzipped() { + # Ensure the file exists and is not empty before checking + if [ ! -s "\\$1" ]; then + return 1 + fi + # Get the MIME type of the file. The '-b' option omits the filename from the output. + local mime_type + mime_type=\\$(file -b --mime-type "\\$1") + + # Check if the MIME type corresponds to gzip. + # application/gzip is standard, while application/x-gzip is also commonly seen. + if [[ "\\$mime_type" == "application/gzip" || "\\$mime_type" == "application/x-gzip" ]]; then + return 0 # 0 indicates success (true in bash) + else + return 1 # 1 indicates failure (false in bash) + fi } -trap clean_up EXIT -par_input="\\$(echo "\\$par_input" | tr ';' ' ')" +# Read the ;-separated file paths from the input variable into an array +IFS=";" read -ra input_files <<< "\\$par_input" -echo -n ">> Check if input is gzipped... " -set +eo pipefail -file \\$par_input | grep -q 'gzip' -is_zipped="\\$?" -set -euo pipefail -[[ "\\$is_zipped" == "0" ]] && echo "yes" || echo "no" +# Process the files if the array contains any paths +if [ \\${#input_files[@]} -gt 0 ] && [ -n "\\${input_files[0]}" ]; then -if [[ "\\$is_zipped" == "0" ]]; then - echo ">> zcat gzipped files" - zcat \\$par_input > \\$TMPDIR/contents + # Ensure the output file is empty before we start + > "\\$par_output" + + echo "Processing files for -> \\$par_output" + + # Create a subshell for the loop to group all cat/zcat output. + ( + for file in "\\${input_files[@]}"; do + if [ -z "\\$file" ]; then continue; fi # Skip empty entries in the array + + if is_gzipped "\\$file"; then + zcat "\\$file" + else + cat "\\$file" + fi + done + ) | if [ "\\$par_compress_output" = "true" ]; then + # If compression is enabled, pipe the entire stream to gzip + gzip -c >> "\\$par_output" + else + # Otherwise, just redirect the stream to the plain text file + cat >> "\\$par_output" + fi + + echo "Finished creating \\$par_output." else - echo ">> cat plain files" - cat \\$par_input > \\$TMPDIR/contents + echo "No input files provided in \\\\\\$par_input. Exiting." fi -if [ "\\$par_gzip_output" == true ]; then - echo ">> Zip output file" - gzip \\$TMPDIR/contents - mv \\$TMPDIR/contents.gz \\$par_output -else - mv \\$TMPDIR/contents \\$par_output -fi +echo "Script finished successfully." VIASHMAIN bash "$tempscript" ''' diff --git a/target/nextflow/concat_text/nextflow.config b/target/nextflow/concat_text/nextflow.config index a180dd4..e433c9a 100644 --- a/target/nextflow/concat_text/nextflow.config +++ b/target/nextflow/concat_text/nextflow.config @@ -4,7 +4,7 @@ manifest { nextflowVersion = '!>=20.12.1-edge' version = 'main' description = 'Concatenate a number of text files, handle gzipped text files gracefully and\noptionally gzip the output text file.\n\nThis component is useful for concatening fastq files from different lanes, for instance.\n' - author = 'Toni Verbeiren, Dries Schaumont' + author = 'Toni Verbeiren, Dries Schaumont, Robrecht Cannoodt' } process.container = 'nextflow/bash:latest' diff --git a/target/nextflow/concat_text/nextflow_schema.json b/target/nextflow/concat_text/nextflow_schema.json index f38d953..53f9db8 100644 --- a/target/nextflow/concat_text/nextflow_schema.json +++ b/target/nextflow/concat_text/nextflow_schema.json @@ -17,7 +17,7 @@ "format": "path", "exists": true, "description": "A list of (gzipped) text files.", - "help_text": "Type: `file`, multiple: `True`, required, direction: `input`, example: `[\"input?.txt.gz\"]`. " + "help_text": "Type: `file`, multiple: `True`, required, direction: `input`, example: `[\"input.txt.gz\"]`. " } } }, @@ -35,7 +35,7 @@ "output": { "type": "string", "format": "path", - "description": "File to write the output to, optionally gzipped.", + "description": "File to write the output to, potentially gzipped.", "help_text": "Type: `file`, multiple: `False`, default: `\"$id.$key.output.txt\"`, direction: `output`, example: `\"output.txt\"`. ", "default": "$id.$key.output.txt" } diff --git a/target/nextflow/csv2fasta/.config.vsh.yaml b/target/nextflow/csv2fasta/.config.vsh.yaml index bb7f1d2..593d664 100644 --- a/target/nextflow/csv2fasta/.config.vsh.yaml +++ b/target/nextflow/csv2fasta/.config.vsh.yaml @@ -256,8 +256,8 @@ build_info: engine: "docker|native" output: "target/nextflow/csv2fasta" executable: "target/nextflow/csv2fasta/main.nf" - viash_version: "0.9.4" - git_commit: "32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550" + viash_version: "0.9.7" + git_commit: "b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1" git_remote: "https://github.com/viash-hub/craftbox" package_config: name: "craftbox" @@ -277,7 +277,7 @@ package_config: \ to ensure dependency management and a consistent, reproducible runtime environment.\n\ \ * Unit tested where applicable to ensure components function as expected.\n" info: null - viash_version: "0.9.4" + viash_version: "0.9.7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/csv2fasta/main.nf b/target/nextflow/csv2fasta/main.nf index 19bf3da..f90a105 100644 --- a/target/nextflow/csv2fasta/main.nf +++ b/target/nextflow/csv2fasta/main.nf @@ -1,6 +1,6 @@ // csv2fasta main // -// This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative +// This wrapper script is auto-generated by viash 0.9.7 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data // Intuitive. // @@ -1354,47 +1354,42 @@ def readCsv(file_path) { def splitRegex = java.util.regex.Pattern.compile(''',(?=(?:[^"]*"[^"]*")*[^"]*$)''') def removeQuote = java.util.regex.Pattern.compile('''"(.*)"''') - def br = java.nio.file.Files.newBufferedReader(inputFile) + java.nio.file.Files.newBufferedReader(inputFile).withCloseable { br -> + def row = 0 + def header = null + def line - def row = -1 - def header = null - while (br.ready() && header == null) { - def line = br.readLine() - row++ - if (!line.startsWith("#")) { - header = splitRegex.split(line, -1).collect{field -> - m = removeQuote.matcher(field) - m.find() ? m.replaceFirst('$1') : field - } - } - } - assert header != null: "CSV file should contain a header" - - while (br.ready()) { - def line = br.readLine() - row++ - if (line == null) { - br.close() - break - } - - if (!line.startsWith("#")) { - def predata = splitRegex.split(line, -1) - def data = predata.collect{field -> - if (field == "") { - return null - } - def m = removeQuote.matcher(field) - if (m.find()) { - return m.replaceFirst('$1') - } else { - return field + while (header == null && (line = br.readLine()) != null) { + if (!line.startsWith("#")) { + header = splitRegex.split(line, -1).collect { field -> + def m = removeQuote.matcher(field) + m.find() ? m.replaceFirst('$1') : field } } - assert header.size() == data.size(): "Row $row should contain the same number as fields as the header" - - def dataMap = [header, data].transpose().collectEntries().findAll{it.value != null} - output.add(dataMap) + row++ + } + assert header != null : "CSV file should contain a header" + + while ((line = br.readLine()) != null) { + row++ + if (!line.startsWith("#")) { + def predata = splitRegex.split(line, -1) + def data = predata.collect { field -> + if (field == "") { + return null + } + def m = removeQuote.matcher(field) + if (m.find()) { + return m.replaceFirst('$1') + } else { + return field + } + } + assert header.size() == data.size() : "Row $row should contain the same number as fields as the header" + + def dataMap = [header, data].transpose().collectEntries().findAll { it.value != null } + output.add(dataMap) + } } } @@ -1710,10 +1705,25 @@ process publishFilesProc { ] .transpose() .collectMany{infile, outfile -> - if (infile.toString() != outfile.toString()) { + def infileString = infile.toString() + def outfileString = outfile.toString() + if (infileString != outfileString) { + /* Trailing slashes are removed from both the source and destination arguments. + From source arguments, this is useful when a source argument may have a trailing slash + and specify a symbolic link to a directory. Without removing the slash, cp will dereference + the symbolic link. + See https://www.gnu.org/software/coreutils/manual/html_node/Trailing-slashes.html#Trailing-slashes-1 + + For the destination path addding a trailing slash is a problem when publishing directories: + it requires the destination directory to exist. This fails because we only create the parent + directories first. + */ + def regexTrailingSlashes = ~/\/+$/ + def infileNoTrailingSlash = infileString - regexTrailingSlashes + def outfileNoTrailingSlash = outfileString - regexTrailingSlashes [ - "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"", - "cp -r '${infile.toString()}' '${outfile.toString()}'" + "[ -d \"\$(dirname '${outfileNoTrailingSlash}')\" ] || mkdir -p \"\$(dirname '${outfileNoTrailingSlash}')\"", + "cp -a '${infileNoTrailingSlash}' '${outfileNoTrailingSlash}'" ] } else { // no need to copy if infile is the same as outfile @@ -3351,8 +3361,8 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/csv2fasta", - "viash_version" : "0.9.4", - "git_commit" : "32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550", + "viash_version" : "0.9.7", + "git_commit" : "b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1", "git_remote" : "https://github.com/viash-hub/craftbox" }, "package_config" : { @@ -3360,7 +3370,7 @@ meta = [ "version" : "main", "summary" : "A collection of custom-tailored scripts and applied utilities built with Viash.\n", "description" : "`craftbox` is a curated collection of custom scripts and utilities designed to tackle context-specific tasks.\n\nEmphasizing the Viash principles, `craftbox` components aim for **reusability**, **reproducibility**, and adherence to **best practices**. Key features generally include:\n\n* **Standalone & Nextflow Ready:** Components are built to run directly via the command line or be smoothly integrated into Nextflow workflows.\n* **Custom Implementations:** Contains scripts and tools developed for particular tasks that may not be found in broader collections.\n* **High Quality Standards (promoted by Viash):**\n * Clear documentation for components and their parameters.\n * Full exposure of underlying script/tool arguments for fine-grained control.\n * Containerized (Docker) to ensure dependency management and a consistent, reproducible runtime environment.\n * Unit tested where applicable to ensure components function as expected.\n", - "viash_version" : "0.9.4", + "viash_version" : "0.9.7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/move_files_to_directory/.config.vsh.yaml b/target/nextflow/move_files_to_directory/.config.vsh.yaml index a1f8a3d..ef89b8b 100644 --- a/target/nextflow/move_files_to_directory/.config.vsh.yaml +++ b/target/nextflow/move_files_to_directory/.config.vsh.yaml @@ -14,11 +14,12 @@ authors: href: "https://www.data-intuitive.com" role: "Data Scientist" argument_groups: -- name: "Arguments" +- name: "Inputs" arguments: - type: "file" name: "--input" - description: "Paths of the files that will be copied into the output directory." + description: "Paths of the files or directories that will be copied into the output\ + \ directory." info: null must_exist: true create_parent: true @@ -26,9 +27,11 @@ argument_groups: direction: "input" multiple: true multiple_sep: ";" +- name: "Outputs" + arguments: - type: "file" name: "--output" - description: "Path to output directory" + description: "Path to output directory." info: null must_exist: true create_parent: true @@ -36,20 +39,22 @@ argument_groups: direction: "output" multiple: false multiple_sep: ";" +- name: "Options" + arguments: - type: "boolean_true" name: "--keep_symbolic_links" - alternatives: - - "-d" - description: "Preserve symbolic links." + description: "When set, symbolic links are preserved as symbolic links in the\ + \ output directory. By default, symbolic links are dereferenced and the target\ + \ file is copied." info: null direction: "input" resources: - type: "bash_script" path: "script.sh" is_executable: true -summary: "Publish one or multiple files to the same directory" -description: "This component copies one or multiple files to the same destination\ - \ directory, creating the output directory if it doesn't exist." +summary: "Publish one or multiple files or directories to the same output directory." +description: "This component copies one or multiple files or directories\nto the same\ + \ destination directory, creating the output directory if it doesn't\nexist.\n" test_resources: - type: "bash_script" path: "test.sh" @@ -146,14 +151,16 @@ engines: cmd: null - type: "native" id: "native" +- type: "native" + id: "native" build_info: config: "src/move_files_to_directory/config.vsh.yaml" runner: "nextflow" - engine: "docker|native" + engine: "docker|native|native" output: "target/nextflow/move_files_to_directory" executable: "target/nextflow/move_files_to_directory/main.nf" - viash_version: "0.9.4" - git_commit: "32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550" + viash_version: "0.9.7" + git_commit: "b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1" git_remote: "https://github.com/viash-hub/craftbox" package_config: name: "craftbox" @@ -173,7 +180,7 @@ package_config: \ to ensure dependency management and a consistent, reproducible runtime environment.\n\ \ * Unit tested where applicable to ensure components function as expected.\n" info: null - viash_version: "0.9.4" + viash_version: "0.9.7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/move_files_to_directory/main.nf b/target/nextflow/move_files_to_directory/main.nf index bd367b2..00fbc09 100644 --- a/target/nextflow/move_files_to_directory/main.nf +++ b/target/nextflow/move_files_to_directory/main.nf @@ -1,6 +1,6 @@ // move_files_to_directory main // -// This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative +// This wrapper script is auto-generated by viash 0.9.7 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data // Intuitive. // @@ -1353,47 +1353,42 @@ def readCsv(file_path) { def splitRegex = java.util.regex.Pattern.compile(''',(?=(?:[^"]*"[^"]*")*[^"]*$)''') def removeQuote = java.util.regex.Pattern.compile('''"(.*)"''') - def br = java.nio.file.Files.newBufferedReader(inputFile) + java.nio.file.Files.newBufferedReader(inputFile).withCloseable { br -> + def row = 0 + def header = null + def line - def row = -1 - def header = null - while (br.ready() && header == null) { - def line = br.readLine() - row++ - if (!line.startsWith("#")) { - header = splitRegex.split(line, -1).collect{field -> - m = removeQuote.matcher(field) - m.find() ? m.replaceFirst('$1') : field - } - } - } - assert header != null: "CSV file should contain a header" - - while (br.ready()) { - def line = br.readLine() - row++ - if (line == null) { - br.close() - break - } - - if (!line.startsWith("#")) { - def predata = splitRegex.split(line, -1) - def data = predata.collect{field -> - if (field == "") { - return null - } - def m = removeQuote.matcher(field) - if (m.find()) { - return m.replaceFirst('$1') - } else { - return field + while (header == null && (line = br.readLine()) != null) { + if (!line.startsWith("#")) { + header = splitRegex.split(line, -1).collect { field -> + def m = removeQuote.matcher(field) + m.find() ? m.replaceFirst('$1') : field } } - assert header.size() == data.size(): "Row $row should contain the same number as fields as the header" - - def dataMap = [header, data].transpose().collectEntries().findAll{it.value != null} - output.add(dataMap) + row++ + } + assert header != null : "CSV file should contain a header" + + while ((line = br.readLine()) != null) { + row++ + if (!line.startsWith("#")) { + def predata = splitRegex.split(line, -1) + def data = predata.collect { field -> + if (field == "") { + return null + } + def m = removeQuote.matcher(field) + if (m.find()) { + return m.replaceFirst('$1') + } else { + return field + } + } + assert header.size() == data.size() : "Row $row should contain the same number as fields as the header" + + def dataMap = [header, data].transpose().collectEntries().findAll { it.value != null } + output.add(dataMap) + } } } @@ -1709,10 +1704,25 @@ process publishFilesProc { ] .transpose() .collectMany{infile, outfile -> - if (infile.toString() != outfile.toString()) { + def infileString = infile.toString() + def outfileString = outfile.toString() + if (infileString != outfileString) { + /* Trailing slashes are removed from both the source and destination arguments. + From source arguments, this is useful when a source argument may have a trailing slash + and specify a symbolic link to a directory. Without removing the slash, cp will dereference + the symbolic link. + See https://www.gnu.org/software/coreutils/manual/html_node/Trailing-slashes.html#Trailing-slashes-1 + + For the destination path addding a trailing slash is a problem when publishing directories: + it requires the destination directory to exist. This fails because we only create the parent + directories first. + */ + def regexTrailingSlashes = ~/\/+$/ + def infileNoTrailingSlash = infileString - regexTrailingSlashes + def outfileNoTrailingSlash = outfileString - regexTrailingSlashes [ - "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"", - "cp -r '${infile.toString()}' '${outfile.toString()}'" + "[ -d \"\$(dirname '${outfileNoTrailingSlash}')\" ] || mkdir -p \"\$(dirname '${outfileNoTrailingSlash}')\"", + "cp -a '${infileNoTrailingSlash}' '${outfileNoTrailingSlash}'" ] } else { // no need to copy if infile is the same as outfile @@ -3059,37 +3069,44 @@ meta = [ ], "argument_groups" : [ { - "name" : "Arguments", + "name" : "Inputs", "arguments" : [ { "type" : "file", "name" : "--input", - "description" : "Paths of the files that will be copied into the output directory.", + "description" : "Paths of the files or directories that will be copied into the output directory.", "must_exist" : true, "create_parent" : true, "required" : true, "direction" : "input", "multiple" : true, "multiple_sep" : ";" - }, + } + ] + }, + { + "name" : "Outputs", + "arguments" : [ { "type" : "file", "name" : "--output", - "description" : "Path to output directory", + "description" : "Path to output directory.", "must_exist" : true, "create_parent" : true, "required" : true, "direction" : "output", "multiple" : false, "multiple_sep" : ";" - }, + } + ] + }, + { + "name" : "Options", + "arguments" : [ { "type" : "boolean_true", "name" : "--keep_symbolic_links", - "alternatives" : [ - "-d" - ], - "description" : "Preserve symbolic links.", + "description" : "When set, symbolic links are preserved as symbolic links in the output directory. By default, symbolic links are dereferenced and the target file is copied.", "direction" : "input" } ] @@ -3102,8 +3119,8 @@ meta = [ "is_executable" : true } ], - "summary" : "Publish one or multiple files to the same directory", - "description" : "This component copies one or multiple files to the same destination directory, creating the output directory if it doesn't exist.", + "summary" : "Publish one or multiple files or directories to the same output directory.", + "description" : "This component copies one or multiple files or directories\nto the same destination directory, creating the output directory if it doesn't\nexist.\n", "test_resources" : [ { "type" : "bash_script", @@ -3217,6 +3234,10 @@ meta = [ } ] }, + { + "type" : "native", + "id" : "native" + }, { "type" : "native", "id" : "native" @@ -3225,10 +3246,10 @@ meta = [ "build_info" : { "config" : "/workdir/root/repo/src/move_files_to_directory/config.vsh.yaml", "runner" : "nextflow", - "engine" : "docker|native", + "engine" : "docker|native|native", "output" : "target/nextflow/move_files_to_directory", - "viash_version" : "0.9.4", - "git_commit" : "32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550", + "viash_version" : "0.9.7", + "git_commit" : "b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1", "git_remote" : "https://github.com/viash-hub/craftbox" }, "package_config" : { @@ -3236,7 +3257,7 @@ meta = [ "version" : "main", "summary" : "A collection of custom-tailored scripts and applied utilities built with Viash.\n", "description" : "`craftbox` is a curated collection of custom scripts and utilities designed to tackle context-specific tasks.\n\nEmphasizing the Viash principles, `craftbox` components aim for **reusability**, **reproducibility**, and adherence to **best practices**. Key features generally include:\n\n* **Standalone & Nextflow Ready:** Components are built to run directly via the command line or be smoothly integrated into Nextflow workflows.\n* **Custom Implementations:** Contains scripts and tools developed for particular tasks that may not be found in broader collections.\n* **High Quality Standards (promoted by Viash):**\n * Clear documentation for components and their parameters.\n * Full exposure of underlying script/tool arguments for fine-grained control.\n * Containerized (Docker) to ensure dependency management and a consistent, reproducible runtime environment.\n * Unit tested where applicable to ensure components function as expected.\n", - "viash_version" : "0.9.4", + "viash_version" : "0.9.7", "source" : "src", "target" : "target", "config_mods" : [ @@ -3300,27 +3321,33 @@ $( if [ ! -z ${VIASH_META_MEMORY_PIB+x} ]; then echo "${VIASH_META_MEMORY_PIB}" ## VIASH END - if [[ ! -d "\\$par_output" ]]; then mkdir -p "\\$par_output" fi -extra_params=( ) - -if [ "\\$par_keep_symbolic_links" == "true" ]; then - extra_params+=( "-d" ) +# Set copy flags based on options +if [[ "\\$par_keep_symbolic_links" == "true" ]]; then + # -a implies -dR --preserve=all + # with -d: same as --no-dereference (and --preserve=links) + # and --no-dereference: never follow symbolic links in SOURCE + # and -R, -r, --recursive: copy directories recursively + # --keep-directory-symlink: if the destination already exists and is a + # symbolic link to a directory, follow the symlink and copy into the directory + # it points to, instead of removing the symlink and creating a real directory in its place. + cp_flags="-a --keep-directory-symlink" +else + # -L: always follow symbolic links in SOURCE + cp_flags="-Lr --preserve=all --no-preserve=link --keep-directory-symlink" fi -# Process multiple input files -IFS=";" read -ra input_files <<< "\\$par_input" -for file in "\\${input_files[@]}"; do - # Check if the file exists before copying - if [[ -f "\\$file" ]]; then - - cp \\${extra_params[@]} "\\$file" "\\$par_output/" - echo "Copied \\$file to \\$par_output/" +# Process multiple input paths (files or directories) +IFS=";" read -ra input_paths <<< "\\$par_input" +for path in "\\${input_paths[@]}"; do + if [[ -e "\\$path" ]] || [[ -L "\\$path" ]]; then + cp \\$cp_flags "\\$path" "\\$par_output/" + echo "Copied \\$path to \\$par_output/" else - echo "Warning: Input file \\$file does not exist, skipping" + echo "Warning: Input path \\$path does not exist, skipping" fi done VIASHMAIN diff --git a/target/nextflow/move_files_to_directory/nextflow.config b/target/nextflow/move_files_to_directory/nextflow.config index ab2ac11..570a9bf 100644 --- a/target/nextflow/move_files_to_directory/nextflow.config +++ b/target/nextflow/move_files_to_directory/nextflow.config @@ -3,7 +3,7 @@ manifest { mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' version = 'main' - description = 'This component copies one or multiple files to the same destination directory, creating the output directory if it doesn\'t exist.' + description = 'This component copies one or multiple files or directories\nto the same destination directory, creating the output directory if it doesn\'t\nexist.\n' author = 'Dorien Roosen' } diff --git a/target/nextflow/move_files_to_directory/nextflow_schema.json b/target/nextflow/move_files_to_directory/nextflow_schema.json index 7ea5f60..ec3370c 100644 --- a/target/nextflow/move_files_to_directory/nextflow_schema.json +++ b/target/nextflow/move_files_to_directory/nextflow_schema.json @@ -1,11 +1,11 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "move_files_to_directory", - "description": "This component copies one or multiple files to the same destination directory, creating the output directory if it doesn't exist.", + "description": "This component copies one or multiple files or directories\nto the same destination directory, creating the output directory if it doesn't\nexist.\n", "type": "object", "$defs": { - "arguments": { - "title": "Arguments", + "inputs": { + "title": "Inputs", "type": "object", "description": "No description", "properties": { @@ -16,19 +16,33 @@ }, "format": "path", "exists": true, - "description": "Paths of the files that will be copied into the output directory.", + "description": "Paths of the files or directories that will be copied into the output directory.", "help_text": "Type: `file`, multiple: `True`, required, direction: `input`. " - }, + } + } + }, + "outputs": { + "title": "Outputs", + "type": "object", + "description": "No description", + "properties": { "output": { "type": "string", "format": "path", - "description": "Path to output directory", + "description": "Path to output directory.", "help_text": "Type: `file`, multiple: `False`, required, default: `\"$id.$key.output\"`, direction: `output`. ", "default": "$id.$key.output" - }, + } + } + }, + "options": { + "title": "Options", + "type": "object", + "description": "No description", + "properties": { "keep_symbolic_links": { "type": "boolean", - "description": "Preserve symbolic links.", + "description": "When set, symbolic links are preserved as symbolic links in the output directory", "help_text": "Type: `boolean_true`, multiple: `False`, default: `false`. ", "default": false } @@ -49,7 +63,13 @@ }, "allOf": [ { - "$ref": "#/$defs/arguments" + "$ref": "#/$defs/inputs" + }, + { + "$ref": "#/$defs/outputs" + }, + { + "$ref": "#/$defs/options" }, { "$ref": "#/$defs/nextflow input-output arguments" diff --git a/target/nextflow/sync_resources/.config.vsh.yaml b/target/nextflow/sync_resources/.config.vsh.yaml index f5da0d3..f187820 100644 --- a/target/nextflow/sync_resources/.config.vsh.yaml +++ b/target/nextflow/sync_resources/.config.vsh.yaml @@ -196,8 +196,8 @@ build_info: engine: "docker|native" output: "target/nextflow/sync_resources" executable: "target/nextflow/sync_resources/main.nf" - viash_version: "0.9.4" - git_commit: "32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550" + viash_version: "0.9.7" + git_commit: "b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1" git_remote: "https://github.com/viash-hub/craftbox" package_config: name: "craftbox" @@ -217,7 +217,7 @@ package_config: \ to ensure dependency management and a consistent, reproducible runtime environment.\n\ \ * Unit tested where applicable to ensure components function as expected.\n" info: null - viash_version: "0.9.4" + viash_version: "0.9.7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/sync_resources/main.nf b/target/nextflow/sync_resources/main.nf index 97193d0..4d51f8c 100644 --- a/target/nextflow/sync_resources/main.nf +++ b/target/nextflow/sync_resources/main.nf @@ -1,6 +1,6 @@ // sync_resources main // -// This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative +// This wrapper script is auto-generated by viash 0.9.7 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data // Intuitive. // @@ -1354,47 +1354,42 @@ def readCsv(file_path) { def splitRegex = java.util.regex.Pattern.compile(''',(?=(?:[^"]*"[^"]*")*[^"]*$)''') def removeQuote = java.util.regex.Pattern.compile('''"(.*)"''') - def br = java.nio.file.Files.newBufferedReader(inputFile) + java.nio.file.Files.newBufferedReader(inputFile).withCloseable { br -> + def row = 0 + def header = null + def line - def row = -1 - def header = null - while (br.ready() && header == null) { - def line = br.readLine() - row++ - if (!line.startsWith("#")) { - header = splitRegex.split(line, -1).collect{field -> - m = removeQuote.matcher(field) - m.find() ? m.replaceFirst('$1') : field - } - } - } - assert header != null: "CSV file should contain a header" - - while (br.ready()) { - def line = br.readLine() - row++ - if (line == null) { - br.close() - break - } - - if (!line.startsWith("#")) { - def predata = splitRegex.split(line, -1) - def data = predata.collect{field -> - if (field == "") { - return null - } - def m = removeQuote.matcher(field) - if (m.find()) { - return m.replaceFirst('$1') - } else { - return field + while (header == null && (line = br.readLine()) != null) { + if (!line.startsWith("#")) { + header = splitRegex.split(line, -1).collect { field -> + def m = removeQuote.matcher(field) + m.find() ? m.replaceFirst('$1') : field } } - assert header.size() == data.size(): "Row $row should contain the same number as fields as the header" - - def dataMap = [header, data].transpose().collectEntries().findAll{it.value != null} - output.add(dataMap) + row++ + } + assert header != null : "CSV file should contain a header" + + while ((line = br.readLine()) != null) { + row++ + if (!line.startsWith("#")) { + def predata = splitRegex.split(line, -1) + def data = predata.collect { field -> + if (field == "") { + return null + } + def m = removeQuote.matcher(field) + if (m.find()) { + return m.replaceFirst('$1') + } else { + return field + } + } + assert header.size() == data.size() : "Row $row should contain the same number as fields as the header" + + def dataMap = [header, data].transpose().collectEntries().findAll { it.value != null } + output.add(dataMap) + } } } @@ -1710,10 +1705,25 @@ process publishFilesProc { ] .transpose() .collectMany{infile, outfile -> - if (infile.toString() != outfile.toString()) { + def infileString = infile.toString() + def outfileString = outfile.toString() + if (infileString != outfileString) { + /* Trailing slashes are removed from both the source and destination arguments. + From source arguments, this is useful when a source argument may have a trailing slash + and specify a symbolic link to a directory. Without removing the slash, cp will dereference + the symbolic link. + See https://www.gnu.org/software/coreutils/manual/html_node/Trailing-slashes.html#Trailing-slashes-1 + + For the destination path addding a trailing slash is a problem when publishing directories: + it requires the destination directory to exist. This fails because we only create the parent + directories first. + */ + def regexTrailingSlashes = ~/\/+$/ + def infileNoTrailingSlash = infileString - regexTrailingSlashes + def outfileNoTrailingSlash = outfileString - regexTrailingSlashes [ - "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"", - "cp -r '${infile.toString()}' '${outfile.toString()}'" + "[ -d \"\$(dirname '${outfileNoTrailingSlash}')\" ] || mkdir -p \"\$(dirname '${outfileNoTrailingSlash}')\"", + "cp -a '${infileNoTrailingSlash}' '${outfileNoTrailingSlash}'" ] } else { // no need to copy if infile is the same as outfile @@ -3293,8 +3303,8 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/sync_resources", - "viash_version" : "0.9.4", - "git_commit" : "32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550", + "viash_version" : "0.9.7", + "git_commit" : "b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1", "git_remote" : "https://github.com/viash-hub/craftbox" }, "package_config" : { @@ -3302,7 +3312,7 @@ meta = [ "version" : "main", "summary" : "A collection of custom-tailored scripts and applied utilities built with Viash.\n", "description" : "`craftbox` is a curated collection of custom scripts and utilities designed to tackle context-specific tasks.\n\nEmphasizing the Viash principles, `craftbox` components aim for **reusability**, **reproducibility**, and adherence to **best practices**. Key features generally include:\n\n* **Standalone & Nextflow Ready:** Components are built to run directly via the command line or be smoothly integrated into Nextflow workflows.\n* **Custom Implementations:** Contains scripts and tools developed for particular tasks that may not be found in broader collections.\n* **High Quality Standards (promoted by Viash):**\n * Clear documentation for components and their parameters.\n * Full exposure of underlying script/tool arguments for fine-grained control.\n * Containerized (Docker) to ensure dependency management and a consistent, reproducible runtime environment.\n * Unit tested where applicable to ensure components function as expected.\n", - "viash_version" : "0.9.4", + "viash_version" : "0.9.7", "source" : "src", "target" : "target", "config_mods" : [ diff --git a/target/nextflow/untar/.config.vsh.yaml b/target/nextflow/untar/.config.vsh.yaml index 87391a6..f549de1 100644 --- a/target/nextflow/untar/.config.vsh.yaml +++ b/target/nextflow/untar/.config.vsh.yaml @@ -181,8 +181,8 @@ build_info: engine: "docker|native" output: "target/nextflow/untar" executable: "target/nextflow/untar/main.nf" - viash_version: "0.9.4" - git_commit: "32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550" + viash_version: "0.9.7" + git_commit: "b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1" git_remote: "https://github.com/viash-hub/craftbox" package_config: name: "craftbox" @@ -202,7 +202,7 @@ package_config: \ to ensure dependency management and a consistent, reproducible runtime environment.\n\ \ * Unit tested where applicable to ensure components function as expected.\n" info: null - viash_version: "0.9.4" + viash_version: "0.9.7" source: "src" target: "target" config_mods: diff --git a/target/nextflow/untar/main.nf b/target/nextflow/untar/main.nf index 59cd817..11762ab 100644 --- a/target/nextflow/untar/main.nf +++ b/target/nextflow/untar/main.nf @@ -1,6 +1,6 @@ // untar main // -// This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative +// This wrapper script is auto-generated by viash 0.9.7 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data // Intuitive. // @@ -1354,47 +1354,42 @@ def readCsv(file_path) { def splitRegex = java.util.regex.Pattern.compile(''',(?=(?:[^"]*"[^"]*")*[^"]*$)''') def removeQuote = java.util.regex.Pattern.compile('''"(.*)"''') - def br = java.nio.file.Files.newBufferedReader(inputFile) + java.nio.file.Files.newBufferedReader(inputFile).withCloseable { br -> + def row = 0 + def header = null + def line - def row = -1 - def header = null - while (br.ready() && header == null) { - def line = br.readLine() - row++ - if (!line.startsWith("#")) { - header = splitRegex.split(line, -1).collect{field -> - m = removeQuote.matcher(field) - m.find() ? m.replaceFirst('$1') : field - } - } - } - assert header != null: "CSV file should contain a header" - - while (br.ready()) { - def line = br.readLine() - row++ - if (line == null) { - br.close() - break - } - - if (!line.startsWith("#")) { - def predata = splitRegex.split(line, -1) - def data = predata.collect{field -> - if (field == "") { - return null - } - def m = removeQuote.matcher(field) - if (m.find()) { - return m.replaceFirst('$1') - } else { - return field + while (header == null && (line = br.readLine()) != null) { + if (!line.startsWith("#")) { + header = splitRegex.split(line, -1).collect { field -> + def m = removeQuote.matcher(field) + m.find() ? m.replaceFirst('$1') : field } } - assert header.size() == data.size(): "Row $row should contain the same number as fields as the header" - - def dataMap = [header, data].transpose().collectEntries().findAll{it.value != null} - output.add(dataMap) + row++ + } + assert header != null : "CSV file should contain a header" + + while ((line = br.readLine()) != null) { + row++ + if (!line.startsWith("#")) { + def predata = splitRegex.split(line, -1) + def data = predata.collect { field -> + if (field == "") { + return null + } + def m = removeQuote.matcher(field) + if (m.find()) { + return m.replaceFirst('$1') + } else { + return field + } + } + assert header.size() == data.size() : "Row $row should contain the same number as fields as the header" + + def dataMap = [header, data].transpose().collectEntries().findAll { it.value != null } + output.add(dataMap) + } } } @@ -1710,10 +1705,25 @@ process publishFilesProc { ] .transpose() .collectMany{infile, outfile -> - if (infile.toString() != outfile.toString()) { + def infileString = infile.toString() + def outfileString = outfile.toString() + if (infileString != outfileString) { + /* Trailing slashes are removed from both the source and destination arguments. + From source arguments, this is useful when a source argument may have a trailing slash + and specify a symbolic link to a directory. Without removing the slash, cp will dereference + the symbolic link. + See https://www.gnu.org/software/coreutils/manual/html_node/Trailing-slashes.html#Trailing-slashes-1 + + For the destination path addding a trailing slash is a problem when publishing directories: + it requires the destination directory to exist. This fails because we only create the parent + directories first. + */ + def regexTrailingSlashes = ~/\/+$/ + def infileNoTrailingSlash = infileString - regexTrailingSlashes + def outfileNoTrailingSlash = outfileString - regexTrailingSlashes [ - "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"", - "cp -r '${infile.toString()}' '${outfile.toString()}'" + "[ -d \"\$(dirname '${outfileNoTrailingSlash}')\" ] || mkdir -p \"\$(dirname '${outfileNoTrailingSlash}')\"", + "cp -a '${infileNoTrailingSlash}' '${outfileNoTrailingSlash}'" ] } else { // no need to copy if infile is the same as outfile @@ -3272,8 +3282,8 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/untar", - "viash_version" : "0.9.4", - "git_commit" : "32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550", + "viash_version" : "0.9.7", + "git_commit" : "b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1", "git_remote" : "https://github.com/viash-hub/craftbox" }, "package_config" : { @@ -3281,7 +3291,7 @@ meta = [ "version" : "main", "summary" : "A collection of custom-tailored scripts and applied utilities built with Viash.\n", "description" : "`craftbox` is a curated collection of custom scripts and utilities designed to tackle context-specific tasks.\n\nEmphasizing the Viash principles, `craftbox` components aim for **reusability**, **reproducibility**, and adherence to **best practices**. Key features generally include:\n\n* **Standalone & Nextflow Ready:** Components are built to run directly via the command line or be smoothly integrated into Nextflow workflows.\n* **Custom Implementations:** Contains scripts and tools developed for particular tasks that may not be found in broader collections.\n* **High Quality Standards (promoted by Viash):**\n * Clear documentation for components and their parameters.\n * Full exposure of underlying script/tool arguments for fine-grained control.\n * Containerized (Docker) to ensure dependency management and a consistent, reproducible runtime environment.\n * Unit tested where applicable to ensure components function as expected.\n", - "viash_version" : "0.9.4", + "viash_version" : "0.9.7", "source" : "src", "target" : "target", "config_mods" : [