Build branch craftbox/main with version main to craftbox on branch main (b74f552)

Build pipeline: vsh-ci-build-template-xfb9n

Source commit: b74f55253e

Source message: Prepare for 0.3.2 release (#19)
This commit is contained in:
CI
2026-06-01 12:59:55 +00:00
parent d23cce7a1f
commit f00709ee19
38 changed files with 943 additions and 587 deletions

View File

@@ -1,3 +1,21 @@
# craftbox 0.3.2
## NEW FEATURES
* `move_files_to_directory`: add support for moving directories (PR #17).
## MINOR CHANGES
* Bump viash to 0.9.7 (PR #18).
# craftbox 0.3.1
## MINOR CHANGES
* `concat_text`: Allow input files to be a mixed bag of gzipped and plain text files (PR #12).
* `concat_text`: Stream concatenated files to gzip (PR #12).
# craftbox 0.3.0
## NEW FEATURES

View File

@@ -43,7 +43,7 @@ Viash components in craftbox can be run in various ways:
``` mermaid lang="mermaid
flowchart TD
A[craftbox v0.3.0] --> B(Viash Hub Launch)
A[craftbox v0.3.2] --> B(Viash Hub Launch)
A --> C(Viash CLI)
A --> D(Nextflow CLI)
A --> E(Seqera Cloud)
@@ -53,7 +53,7 @@ flowchart TD
### 1. Via the Viash Hub Launch interface
You can run this component directly from the Viash Hub [Launch
interface](https://www.viash-hub.com/launch?package=craftbox&version=v0.3.0&component=concat_rtext&runner=Executable).
interface](https://www.viash-hub.com/launch?package=craftbox&version=v0.3.2&component=concat_rtext&runner=Executable).
![](docs/viash-hub.png)
@@ -63,9 +63,9 @@ You can run this component directly from the command line using the
Viash CLI.
``` bash
viash run vsh://craftbox@v0.3.0/concat_rtext -- --help
viash run vsh://craftbox@v0.3.2/concat_rtext -- --help
viash run vsh://craftbox@v0.3.0/concat_rtext -- \
viash run vsh://craftbox@v0.3.2/concat_rtext -- \
--input path/to/input.txt \
--input path/to/compressed.txt.gz \
--output path/to/output.txt
@@ -80,7 +80,7 @@ You can run this component as a Nextflow pipeline.
``` bash
nextflow run https://packages.viash-hub.com/vsh/craftbox \
-revision v0.3.0 \
-revision v0.3.2 \
-main-script target/nextflow/concat_rtext/main.nf \
-latest -resume \
-profile docker \
@@ -109,7 +109,7 @@ component as a dependency:
``` yaml
dependencies:
- name: concat_rtext
repository: vsh://craftbox@v0.3.0
repository: vsh://craftbox@v0.3.2
```
**Tip:** See the [Viash

View File

@@ -7,7 +7,7 @@ license <- paste0(package$links$repository, "/blob/main/LICENSE")
contributing <- paste0(package$links$repository, "/blob/main/CONTRIBUTING.md")
pkg <- package$name
ver <- if (!is.null(package$version)) package$version else "v0.3.0"
ver <- if (!is.null(package$version)) package$version else "v0.3.2"
comp <- "concat_rtext"
```
# 🪡📦 `r pkg`

View File

@@ -18,7 +18,7 @@ license: MIT
links:
issue_tracker: https://github.com/viash-hub/craftbox/issues
repository: https://github.com/viash-hub/craftbox
viash_version: 0.9.4
viash_version: 0.9.7
config_mods: |
.requirements.commands := ['ps']
version: main

View File

@@ -1,5 +1,5 @@
name: concat_text
summary: Concatenate a number of text files
summary: Concatenate multiple (possibly gzipped) text files
description: |
Concatenate a number of text files, handle gzipped text files gracefully and
optionally gzip the output text file.
@@ -11,12 +11,13 @@ authors:
roles: [ author, maintainer ]
- __merge__: /src/_authors/dries_schaumont.yaml
roles: [ reviewer ]
- __merge__: /src/_authors/robrecht_cannoodt.yaml
roles: [ contributor ]
info:
improvements: |
This component could be improved in 2 ways:
1. Allow for a mix of zipped and plain input files
2. Allow to specify a compression algorithm for the output
This component could be improved:
1. Allow to specify a compression algorithm for the output
argument_groups:
- name: Input arguments
@@ -26,14 +27,14 @@ argument_groups:
type: file
multiple: true
required: true
example: input?.txt.gz
example: input.txt.gz
- name: Output arguments
arguments:
- name: "--gzip_output"
type: boolean_true
description: Should the output be zipped?
- name: --output
description: File to write the output to, optionally gzipped.
description: File to write the output to, potentially gzipped.
type: file
direction: output
example: output.txt

View File

@@ -1,34 +1,65 @@
#!/usr/bin/env bash
set -euo pipefail
set -eo pipefail
TMPDIR=$(mktemp -d "$meta_temp_dir/concat_text-XXXXXX")
function clean_up {
[[ -d "$TMPDIR" ]] && rm -r "$TMPDIR"
## VIASH START
par_input="README.md;README.qmd"
par_output="concatenated_output.txt"
par_compress_output="true"
## VIASH END
# --- Function to check for GZIP format using the 'file' command ---
is_gzipped() {
# Ensure the file exists and is not empty before checking
if [ ! -s "$1" ]; then
return 1
fi
# Get the MIME type of the file. The '-b' option omits the filename from the output.
local mime_type
mime_type=$(file -b --mime-type "$1")
# Check if the MIME type corresponds to gzip.
# application/gzip is standard, while application/x-gzip is also commonly seen.
if [[ "$mime_type" == "application/gzip" || "$mime_type" == "application/x-gzip" ]]; then
return 0 # 0 indicates success (true in bash)
else
return 1 # 1 indicates failure (false in bash)
fi
}
trap clean_up EXIT
par_input="$(echo "$par_input" | tr ';' ' ')"
# Read the ;-separated file paths from the input variable into an array
IFS=";" read -ra input_files <<< "$par_input"
echo -n ">> Check if input is gzipped... "
set +eo pipefail
file $par_input | grep -q 'gzip'
is_zipped="$?"
set -euo pipefail
[[ "$is_zipped" == "0" ]] && echo "yes" || echo "no"
# Process the files if the array contains any paths
if [ ${#input_files[@]} -gt 0 ] && [ -n "${input_files[0]}" ]; then
if [[ "$is_zipped" == "0" ]]; then
echo ">> zcat gzipped files"
zcat $par_input > $TMPDIR/contents
# Ensure the output file is empty before we start
> "$par_output"
echo "Processing files for -> $par_output"
# Create a subshell for the loop to group all cat/zcat output.
(
for file in "${input_files[@]}"; do
if [ -z "$file" ]; then continue; fi # Skip empty entries in the array
if is_gzipped "$file"; then
zcat "$file"
else
cat "$file"
fi
done
) | if [ "$par_compress_output" = "true" ]; then
# If compression is enabled, pipe the entire stream to gzip
gzip -c >> "$par_output"
else
# Otherwise, just redirect the stream to the plain text file
cat >> "$par_output"
fi
echo "Finished creating $par_output."
else
echo ">> cat plain files"
cat $par_input > $TMPDIR/contents
echo "No input files provided in \$par_input. Exiting."
fi
if [ "$par_gzip_output" == true ]; then
echo ">> Zip output file"
gzip $TMPDIR/contents
mv $TMPDIR/contents.gz $par_output
else
mv $TMPDIR/contents $par_output
fi
echo "Script finished successfully."

View File

@@ -2,6 +2,65 @@
set -euo pipefail
echo ">> Running concat_text.sh script"
is_gzipped() {
# Ensure the file exists and is not empty
if [ ! -s "$1" ]; then
return 1
fi
# Get the MIME type of the file
local mime_type
mime_type=$(file -b --mime-type "$1")
# Check if the MIME type corresponds to gzip.
if [[ "$mime_type" == "application/gzip" || "$mime_type" == "application/x-gzip" ]]; then
return 0
else
return 1
fi
}
compare_files() {
local file1="$1"
local file2="$2"
if [[ ! -f "$file1" || ! -f "$file2" ]]; then
echo "One of the files does not exist: $file1 or $file2"
return 1
fi
# decompress file 1 if need be
if is_gzipped "$file1"; then
file1=$(mktemp)
zcat "$1" > "$file1"
fi
# decompress file 2 if need be
if is_gzipped "$file2"; then
file2=$(mktemp)
zcat "$2" > "$file2"
fi
if cmp -s "$file1" "$file2"; then
echo "Files are identical."
return 0
else
echo "Files differ."
echo "Found:"
if is_gzipped "$file1"; then
zcat "$file1" | od -c
else
cat "$file1" | od -c
fi
echo "Expected:"
if is_gzipped "$file2"; then
zcat "$file2" | od -c
else
cat "$file2" | od -c
fi
return 1
fi
}
## TEST RESOURCES
echo ">> Creating test input files file[1-3].txt"
INPUT_FILE_1="file1.txt"
INPUT_FILE_2="file2.txt"
@@ -25,46 +84,31 @@ EOF
gzip -k "expected_output.txt"
## RUN TESTS
echo ">> Run component on 3 plain input files, plain output"
$meta_executable \
--input "$INPUT_FILE_1;$INPUT_FILE_2;$INPUT_FILE_3" \
--output "output1.txt"
compare_files "output1.txt" "expected_output.txt"
[[ ! -f "output1.txt" ]] \
&& echo "Output file output1.txt not found!" && exit 1
[[ $(cmp "output1.txt" "expected_output.txt") ]] \
&& echo "Output file output1.txt is not as expected!" && exit 1
echo ">> Run component on 3 zipped input files, plain output"
echo ">> Run component on mixed input files, plain output"
$meta_executable \
--input "$INPUT_FILE_1.gz;$INPUT_FILE_2.gz;$INPUT_FILE_3.gz" \
--input "$INPUT_FILE_1.gz;$INPUT_FILE_2;$INPUT_FILE_3.gz" \
--output "output2.txt"
[[ ! -f "output2.txt" ]] \
&& echo "Output file output2.txt not found!" && exit 1
[[ $(cmp "output2.txt" "expected_output.txt") ]] \
&& echo "Output file output2.txt is not as expected!" && exit 1
compare_files "output2.txt" "expected_output.txt"
echo ">> Run component on 3 plain input files, zipped output"
$meta_executable \
--input "$INPUT_FILE_1;$INPUT_FILE_2;$INPUT_FILE_3" \
--output "output3.txt.gz" \
--gzip_output
compare_files "output3.txt.gz" "expected_output.txt.gz"
[[ ! -f "output3.txt.gz" ]] \
&& echo "Output file output3.txt.gz not found!" && exit 1
[[ $(cmp "output3.txt.gz" "expected_output.txt.gz") ]] \
&& echo "Output file output3.txt.gz is not as expected!" && exit 1
echo ">> Run component on 3 zipped input files, zipped output"
echo ">> Run component on mixed input files, zipped output"
$meta_executable \
--input "$INPUT_FILE_1.gz;$INPUT_FILE_2.gz;$INPUT_FILE_3.gz" \
--input "$INPUT_FILE_1.gz;$INPUT_FILE_2;$INPUT_FILE_3.gz" \
--output "output4.txt.gz" \
--gzip_output
[[ ! -f "output4.txt.gz" ]] \
&& echo "Output file output4.txt.gz not found!" && exit 1
[[ $(cmp "output4.txt.gz" "expected_output.txt.gz") ]] \
&& echo "Output file output4.txt.gz is not as expected!" && exit 1
compare_files "output4.txt.gz" "expected_output.txt.gz"
echo ">> Tests done"

View File

@@ -1,39 +1,52 @@
name: move_files_to_directory
summary: Publish one or multiple files to the same directory
description: This component copies one or multiple files to the same destination directory, creating the output directory if it doesn't exist.
summary: Publish one or multiple files or directories to the same output directory.
description: |
This component copies one or multiple files or directories
to the same destination directory, creating the output directory if it doesn't
exist.
authors:
- __merge__: /src/_authors/dorien_roosen.yaml
roles: [ maintainer ]
arguments:
- name: "--input"
type: file
direction: input
required: true
multiple: true
description: Paths of the files that will be copied into the output directory.
- name: "--output"
type: file
direction: output
required: true
description: Path to output directory
- name: "--keep_symbolic_links"
alternatives: [-d]
type: boolean_true
description: Preserve symbolic links.
argument_groups:
- name: Inputs
arguments:
- name: "--input"
type: file
direction: input
required: true
multiple: true
description: Paths of the files or directories that will be copied into the output directory.
- name: Outputs
arguments:
- name: "--output"
type: file
direction: output
required: true
description: Path to output directory.
- name: Options
arguments:
- name: "--keep_symbolic_links"
type: boolean_true
description: When set, symbolic links are preserved as symbolic links in the output directory. By default, symbolic links are dereferenced and the target file is copied.
resources:
- type: bash_script
path: script.sh
test_resources:
- type: bash_script
path: test.sh
engines:
- type: docker
image: debian:latest
setup:
- type: docker
image: debian:latest
setup:
- type: apt
packages: [procps]
packages:
- procps
- type: native
runners:
- type: executable
- type: nextflow
- type: executable
- type: nextflow

View File

@@ -3,30 +3,37 @@
set -eo pipefail
## VIASH START
par_input="input.txt;input_2.txt"
par_input="input.txt;input_dir.zarr"
par_output="output"
par_keep_symbolic_links="false"
## VIASH END
if [[ ! -d "$par_output" ]]; then
mkdir -p "$par_output"
fi
extra_params=( )
if [ "$par_keep_symbolic_links" == "true" ]; then
extra_params+=( "-d" )
# Set copy flags based on options
if [[ "$par_keep_symbolic_links" == "true" ]]; then
# -a implies -dR --preserve=all
# with -d: same as --no-dereference (and --preserve=links)
# and --no-dereference: never follow symbolic links in SOURCE
# and -R, -r, --recursive: copy directories recursively
# --keep-directory-symlink: if the destination already exists and is a
# symbolic link to a directory, follow the symlink and copy into the directory
# it points to, instead of removing the symlink and creating a real directory in its place.
cp_flags="-a --keep-directory-symlink"
else
# -L: always follow symbolic links in SOURCE
cp_flags="-Lr --preserve=all --no-preserve=link --keep-directory-symlink"
fi
# Process multiple input files
IFS=";" read -ra input_files <<< "$par_input"
for file in "${input_files[@]}"; do
# Check if the file exists before copying
if [[ -f "$file" ]]; then
cp ${extra_params[@]} "$file" "$par_output/"
echo "Copied $file to $par_output/"
# Process multiple input paths (files or directories)
IFS=";" read -ra input_paths <<< "$par_input"
for path in "${input_paths[@]}"; do
if [[ -e "$path" ]] || [[ -L "$path" ]]; then
cp $cp_flags "$path" "$par_output/"
echo "Copied $path to $par_output/"
else
echo "Warning: Input file $file does not exist, skipping"
echo "Warning: Input path $path does not exist, skipping"
fi
done

View File

@@ -11,7 +11,7 @@ trap clean_up EXIT
touch "$TMPDIR/test_file.txt"
touch "$TMPDIR/another_file.txt"
./move_files_to_directory \
$meta_executable \
--input "$TMPDIR/test_file.txt" \
--input "$TMPDIR/another_file.txt" \
--output "$TMPDIR/test_output"
@@ -19,9 +19,23 @@ touch "$TMPDIR/another_file.txt"
[[ ! -d "$TMPDIR/test_output" ]] && echo "It seems no output directory is generated" && exit 1
[[ ! -f "$TMPDIR/test_output/test_file.txt" ]] && [[ ! -f test_output/another_file.txt ]] && echo "Output files were not copied to the output directory" && exit 1
# Test copying a directory
mkdir -p "$TMPDIR/test_dir"
touch "$TMPDIR/test_dir/file_in_dir.txt"
touch "$TMPDIR/test_dir/another_file_in_dir.txt"
$meta_executable \
--input "$TMPDIR/test_dir" \
--output "$TMPDIR/test_output_dir"
[[ ! -d "$TMPDIR/test_output_dir" ]] && echo "It seems no output directory (test_output_dir) is generated" && exit 1
[[ ! -d "$TMPDIR/test_output_dir/test_dir" ]] && echo "Input directory was not copied to the output directory" && exit 1
[[ ! -f "$TMPDIR/test_output_dir/test_dir/file_in_dir.txt" ]] && echo "Files inside the copied directory are missing" && exit 1
[[ ! -f "$TMPDIR/test_output_dir/test_dir/another_file_in_dir.txt" ]] && echo "Files inside the copied directory are missing" && exit 1
ln -s "$TMPDIR/test_file.txt" "$TMPDIR/symlink.txt"
./move_files_to_directory \
$meta_executable \
--input "$TMPDIR/symlink.txt" \
--output "$TMPDIR/test_output_symlink" \
--keep_symbolic_links

View File

@@ -156,8 +156,8 @@ build_info:
engine: "docker|native"
output: "target/executable/check_disk_space"
executable: "target/executable/check_disk_space/check_disk_space"
viash_version: "0.9.4"
git_commit: "32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550"
viash_version: "0.9.7"
git_commit: "b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1"
git_remote: "https://github.com/viash-hub/craftbox"
package_config:
name: "craftbox"
@@ -177,7 +177,7 @@ package_config:
\ to ensure dependency management and a consistent, reproducible runtime environment.\n\
\ * Unit tested where applicable to ensure components function as expected.\n"
info: null
viash_version: "0.9.4"
viash_version: "0.9.7"
source: "src"
target: "target"
config_mods:

View File

@@ -2,7 +2,7 @@
# check_disk_space main
#
# This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative
# This wrapper script is auto-generated by viash 0.9.7 and is thus a derivative
# work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data
# Intuitive.
#
@@ -446,9 +446,9 @@ function ViashDockerfile {
FROM bash:latest
ENTRYPOINT []
LABEL org.opencontainers.image.description="Companion container for running component check_disk_space"
LABEL org.opencontainers.image.created="2025-09-01T14:32:38Z"
LABEL org.opencontainers.image.created="2026-06-01T12:52:41Z"
LABEL org.opencontainers.image.source="https://github.com/viash-hub/craftbox"
LABEL org.opencontainers.image.revision="32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550"
LABEL org.opencontainers.image.revision="b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1"
LABEL org.opencontainers.image.version="main"
VIASHDOCKER

View File

@@ -26,6 +26,22 @@ authors:
- name: "Data Intuitive"
href: "https://www.data-intuitive.com"
role: "Data Scientist"
- name: "Robrecht Cannoodt"
roles:
- "contributor"
info:
links:
email: "robrecht@data-intuitive.com"
github: "rcannood"
orcid: "0000-0003-3641-729X"
linkedin: "robrechtcannoodt"
organizations:
- name: "Data Intuitive"
href: "https://www.data-intuitive.com"
role: "Data Science Engineer"
- name: "Open Problems"
href: "https://openproblems.bio"
role: "Core Member"
argument_groups:
- name: "Input arguments"
arguments:
@@ -34,7 +50,7 @@ argument_groups:
description: "A list of (gzipped) text files."
info: null
example:
- "input?.txt.gz"
- "input.txt.gz"
must_exist: true
create_parent: true
required: true
@@ -50,7 +66,7 @@ argument_groups:
direction: "input"
- type: "file"
name: "--output"
description: "File to write the output to, optionally gzipped."
description: "File to write the output to, potentially gzipped."
info: null
example:
- "output.txt"
@@ -64,7 +80,7 @@ resources:
- type: "bash_script"
path: "script.sh"
is_executable: true
summary: "Concatenate a number of text files"
summary: "Concatenate multiple (possibly gzipped) text files"
description: "Concatenate a number of text files, handle gzipped text files gracefully\
\ and\noptionally gzip the output text file.\n\nThis component is useful for concatening\
\ fastq files from different lanes, for instance.\n"
@@ -73,9 +89,8 @@ test_resources:
path: "test.sh"
is_executable: true
info:
improvements: "This component could be improved in 2 ways:\n 1. Allow for a mix\
\ of zipped and plain input files\n 2. Allow to specify a compression algorithm\
\ for the output\n"
improvements: "This component could be improved:\n 1. Allow to specify a compression\
\ algorithm for the output\n"
status: "enabled"
scope:
image: "public"
@@ -174,8 +189,8 @@ build_info:
engine: "docker|native"
output: "target/executable/concat_text"
executable: "target/executable/concat_text/concat_text"
viash_version: "0.9.4"
git_commit: "32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550"
viash_version: "0.9.7"
git_commit: "b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1"
git_remote: "https://github.com/viash-hub/craftbox"
package_config:
name: "craftbox"
@@ -195,7 +210,7 @@ package_config:
\ to ensure dependency management and a consistent, reproducible runtime environment.\n\
\ * Unit tested where applicable to ensure components function as expected.\n"
info: null
viash_version: "0.9.4"
viash_version: "0.9.7"
source: "src"
target: "target"
config_mods:

View File

@@ -2,7 +2,7 @@
# concat_text main
#
# This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative
# This wrapper script is auto-generated by viash 0.9.7 and is thus a derivative
# work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data
# Intuitive.
#
@@ -14,6 +14,7 @@
# Component authors:
# * Toni Verbeiren (author, maintainer)
# * Dries Schaumont (reviewer)
# * Robrecht Cannoodt (contributor)
set -e
@@ -451,11 +452,11 @@ FROM alpine:latest
ENTRYPOINT []
RUN apk add --no-cache bash procps file
LABEL org.opencontainers.image.authors="Toni Verbeiren, Dries Schaumont"
LABEL org.opencontainers.image.authors="Toni Verbeiren, Dries Schaumont, Robrecht Cannoodt"
LABEL org.opencontainers.image.description="Companion container for running component concat_text"
LABEL org.opencontainers.image.created="2025-09-01T14:32:38Z"
LABEL org.opencontainers.image.created="2026-06-01T12:52:41Z"
LABEL org.opencontainers.image.source="https://github.com/viash-hub/craftbox"
LABEL org.opencontainers.image.revision="32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550"
LABEL org.opencontainers.image.revision="b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1"
LABEL org.opencontainers.image.version="main"
VIASHDOCKER
@@ -584,7 +585,7 @@ function ViashHelp {
echo "Input arguments:"
echo " --input"
echo " type: file, required parameter, multiple values allowed, file must exist"
echo " example: input?.txt.gz"
echo " example: input.txt.gz"
echo " A list of (gzipped) text files."
echo ""
echo "Output arguments:"
@@ -595,7 +596,7 @@ function ViashHelp {
echo " --output"
echo " type: file, output, file must exist"
echo " example: output.txt"
echo " File to write the output to, optionally gzipped."
echo " File to write the output to, potentially gzipped."
echo ""
echo "Viash built in Computational Requirements:"
echo " ---cpus=INT"
@@ -1080,6 +1081,10 @@ function interrupt {
trap clean_up EXIT
trap interrupt INT SIGINT
cat > "\$tempscript" << 'VIASHMAIN'
#!/usr/bin/env bash
set -eo pipefail
## VIASH START
# The following code has been auto-generated by Viash.
$( if [ ! -z ${VIASH_PAR_INPUT+x} ]; then echo "${VIASH_PAR_INPUT}" | sed "s#'#'\"'\"'#g;s#.*#par_input='&'#" ; else echo "# par_input="; fi )
@@ -1105,40 +1110,62 @@ $( if [ ! -z ${VIASH_META_MEMORY_TIB+x} ]; then echo "${VIASH_META_MEMORY_TIB}"
$( if [ ! -z ${VIASH_META_MEMORY_PIB+x} ]; then echo "${VIASH_META_MEMORY_PIB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_pib='&'#" ; else echo "# meta_memory_pib="; fi )
## VIASH END
#!/usr/bin/env bash
set -euo pipefail
TMPDIR=\$(mktemp -d "\$meta_temp_dir/concat_text-XXXXXX")
function clean_up {
[[ -d "\$TMPDIR" ]] && rm -r "\$TMPDIR"
# --- Function to check for GZIP format using the 'file' command ---
is_gzipped() {
# Ensure the file exists and is not empty before checking
if [ ! -s "\$1" ]; then
return 1
fi
# Get the MIME type of the file. The '-b' option omits the filename from the output.
local mime_type
mime_type=\$(file -b --mime-type "\$1")
# Check if the MIME type corresponds to gzip.
# application/gzip is standard, while application/x-gzip is also commonly seen.
if [[ "\$mime_type" == "application/gzip" || "\$mime_type" == "application/x-gzip" ]]; then
return 0 # 0 indicates success (true in bash)
else
return 1 # 1 indicates failure (false in bash)
fi
}
trap clean_up EXIT
par_input="\$(echo "\$par_input" | tr ';' ' ')"
# Read the ;-separated file paths from the input variable into an array
IFS=";" read -ra input_files <<< "\$par_input"
echo -n ">> Check if input is gzipped... "
set +eo pipefail
file \$par_input | grep -q 'gzip'
is_zipped="\$?"
set -euo pipefail
[[ "\$is_zipped" == "0" ]] && echo "yes" || echo "no"
# Process the files if the array contains any paths
if [ \${#input_files[@]} -gt 0 ] && [ -n "\${input_files[0]}" ]; then
if [[ "\$is_zipped" == "0" ]]; then
echo ">> zcat gzipped files"
zcat \$par_input > \$TMPDIR/contents
# Ensure the output file is empty before we start
> "\$par_output"
echo "Processing files for -> \$par_output"
# Create a subshell for the loop to group all cat/zcat output.
(
for file in "\${input_files[@]}"; do
if [ -z "\$file" ]; then continue; fi # Skip empty entries in the array
if is_gzipped "\$file"; then
zcat "\$file"
else
cat "\$file"
fi
done
) | if [ "\$par_compress_output" = "true" ]; then
# If compression is enabled, pipe the entire stream to gzip
gzip -c >> "\$par_output"
else
# Otherwise, just redirect the stream to the plain text file
cat >> "\$par_output"
fi
echo "Finished creating \$par_output."
else
echo ">> cat plain files"
cat \$par_input > \$TMPDIR/contents
echo "No input files provided in \\\$par_input. Exiting."
fi
if [ "\$par_gzip_output" == true ]; then
echo ">> Zip output file"
gzip \$TMPDIR/contents
mv \$TMPDIR/contents.gz \$par_output
else
mv \$TMPDIR/contents \$par_output
fi
echo "Script finished successfully."
VIASHMAIN
bash "\$tempscript" &
wait "\$!"

View File

@@ -256,8 +256,8 @@ build_info:
engine: "docker|native"
output: "target/executable/csv2fasta"
executable: "target/executable/csv2fasta/csv2fasta"
viash_version: "0.9.4"
git_commit: "32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550"
viash_version: "0.9.7"
git_commit: "b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1"
git_remote: "https://github.com/viash-hub/craftbox"
package_config:
name: "craftbox"
@@ -277,7 +277,7 @@ package_config:
\ to ensure dependency management and a consistent, reproducible runtime environment.\n\
\ * Unit tested where applicable to ensure components function as expected.\n"
info: null
viash_version: "0.9.4"
viash_version: "0.9.7"
source: "src"
target: "target"
config_mods:

View File

@@ -2,7 +2,7 @@
# csv2fasta main
#
# This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative
# This wrapper script is auto-generated by viash 0.9.7 and is thus a derivative
# work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data
# Intuitive.
#
@@ -458,9 +458,9 @@ RUN pip install --upgrade pip && \
LABEL org.opencontainers.image.authors="Dries Schaumont, Robrecht Cannoodt"
LABEL org.opencontainers.image.description="Companion container for running component csv2fasta"
LABEL org.opencontainers.image.created="2025-09-01T14:32:38Z"
LABEL org.opencontainers.image.created="2026-06-01T12:52:41Z"
LABEL org.opencontainers.image.source="https://github.com/viash-hub/craftbox"
LABEL org.opencontainers.image.revision="32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550"
LABEL org.opencontainers.image.revision="b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1"
LABEL org.opencontainers.image.version="main"
VIASHDOCKER

View File

@@ -14,11 +14,12 @@ authors:
href: "https://www.data-intuitive.com"
role: "Data Scientist"
argument_groups:
- name: "Arguments"
- name: "Inputs"
arguments:
- type: "file"
name: "--input"
description: "Paths of the files that will be copied into the output directory."
description: "Paths of the files or directories that will be copied into the output\
\ directory."
info: null
must_exist: true
create_parent: true
@@ -26,9 +27,11 @@ argument_groups:
direction: "input"
multiple: true
multiple_sep: ";"
- name: "Outputs"
arguments:
- type: "file"
name: "--output"
description: "Path to output directory"
description: "Path to output directory."
info: null
must_exist: true
create_parent: true
@@ -36,20 +39,22 @@ argument_groups:
direction: "output"
multiple: false
multiple_sep: ";"
- name: "Options"
arguments:
- type: "boolean_true"
name: "--keep_symbolic_links"
alternatives:
- "-d"
description: "Preserve symbolic links."
description: "When set, symbolic links are preserved as symbolic links in the\
\ output directory. By default, symbolic links are dereferenced and the target\
\ file is copied."
info: null
direction: "input"
resources:
- type: "bash_script"
path: "script.sh"
is_executable: true
summary: "Publish one or multiple files to the same directory"
description: "This component copies one or multiple files to the same destination\
\ directory, creating the output directory if it doesn't exist."
summary: "Publish one or multiple files or directories to the same output directory."
description: "This component copies one or multiple files or directories\nto the same\
\ destination directory, creating the output directory if it doesn't\nexist.\n"
test_resources:
- type: "bash_script"
path: "test.sh"
@@ -146,14 +151,16 @@ engines:
cmd: null
- type: "native"
id: "native"
- type: "native"
id: "native"
build_info:
config: "src/move_files_to_directory/config.vsh.yaml"
runner: "executable"
engine: "docker|native"
engine: "docker|native|native"
output: "target/executable/move_files_to_directory"
executable: "target/executable/move_files_to_directory/move_files_to_directory"
viash_version: "0.9.4"
git_commit: "32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550"
viash_version: "0.9.7"
git_commit: "b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1"
git_remote: "https://github.com/viash-hub/craftbox"
package_config:
name: "craftbox"
@@ -173,7 +180,7 @@ package_config:
\ to ensure dependency management and a consistent, reproducible runtime environment.\n\
\ * Unit tested where applicable to ensure components function as expected.\n"
info: null
viash_version: "0.9.4"
viash_version: "0.9.7"
source: "src"
target: "target"
config_mods:

View File

@@ -2,7 +2,7 @@
# move_files_to_directory main
#
# This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative
# This wrapper script is auto-generated by viash 0.9.7 and is thus a derivative
# work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data
# Intuitive.
#
@@ -454,9 +454,9 @@ RUN apt-get update && \
LABEL org.opencontainers.image.authors="Dorien Roosen"
LABEL org.opencontainers.image.description="Companion container for running component move_files_to_directory"
LABEL org.opencontainers.image.created="2025-09-01T14:32:38Z"
LABEL org.opencontainers.image.created="2026-06-01T12:52:41Z"
LABEL org.opencontainers.image.source="https://github.com/viash-hub/craftbox"
LABEL org.opencontainers.image.revision="32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550"
LABEL org.opencontainers.image.revision="b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1"
LABEL org.opencontainers.image.version="main"
VIASHDOCKER
@@ -576,21 +576,27 @@ VIASH_DOCKER_RUN_ARGS=(-i --rm)
function ViashHelp {
echo "move_files_to_directory main"
echo ""
echo "This component copies one or multiple files to the same destination directory,"
echo "creating the output directory if it doesn't exist."
echo "This component copies one or multiple files or directories"
echo "to the same destination directory, creating the output directory if it doesn't"
echo "exist."
echo ""
echo "Arguments:"
echo "Inputs:"
echo " --input"
echo " type: file, required parameter, multiple values allowed, file must exist"
echo " Paths of the files that will be copied into the output directory."
echo " Paths of the files or directories that will be copied into the output"
echo " directory."
echo ""
echo "Outputs:"
echo " --output"
echo " type: file, required parameter, output, file must exist"
echo " Path to output directory"
echo " Path to output directory."
echo ""
echo " -d, --keep_symbolic_links"
echo "Options:"
echo " --keep_symbolic_links"
echo " type: boolean_true"
echo " Preserve symbolic links."
echo " When set, symbolic links are preserved as symbolic links in the output"
echo " directory. By default, symbolic links are dereferenced and the target"
echo " file is copied."
echo ""
echo "Viash built in Computational Requirements:"
echo " ---cpus=INT"
@@ -613,7 +619,7 @@ function ViashHelp {
echo ""
echo "Viash built in Engines:"
echo " ---engine=ENGINE_ID"
echo " Specify the engine to use. Options are: docker, native."
echo " Specify the engine to use. Options are: docker, native, native."
echo " Default: docker"
}
@@ -675,11 +681,6 @@ while [[ $# -gt 0 ]]; do
VIASH_PAR_KEEP_SYMBOLIC_LINKS=true
shift 1
;;
-d)
[ -n "$VIASH_PAR_KEEP_SYMBOLIC_LINKS" ] && ViashError Bad arguments for option \'-d\': \'$VIASH_PAR_KEEP_SYMBOLIC_LINKS\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1
VIASH_PAR_KEEP_SYMBOLIC_LINKS=true
shift 1
;;
---engine)
VIASH_ENGINE_ID="$2"
shift 2
@@ -753,12 +754,12 @@ done
eval set -- $VIASH_POSITIONAL_ARGS
if [ "$VIASH_ENGINE_ID" == "native" ] ; then
if [ "$VIASH_ENGINE_ID" == "native" ] || [ "$VIASH_ENGINE_ID" == "native" ] ; then
VIASH_ENGINE_TYPE='native'
elif [ "$VIASH_ENGINE_ID" == "docker" ] ; then
VIASH_ENGINE_TYPE='docker'
else
ViashError "Engine '$VIASH_ENGINE_ID' is not recognized. Options are: docker, native."
ViashError "Engine '$VIASH_ENGINE_ID' is not recognized. Options are: docker, native, native."
exit 1
fi
@@ -989,7 +990,7 @@ if [ ! -z "$VIASH_PAR_OUTPUT" ] && [ ! -d "$(dirname "$VIASH_PAR_OUTPUT")" ]; th
mkdir -p "$(dirname "$VIASH_PAR_OUTPUT")"
fi
if [ "$VIASH_ENGINE_ID" == "native" ] ; then
if [ "$VIASH_ENGINE_ID" == "native" ] || [ "$VIASH_ENGINE_ID" == "native" ] ; then
if [ "$VIASH_MODE" == "run" ]; then
VIASH_CMD="bash"
else
@@ -1114,27 +1115,33 @@ $( if [ ! -z ${VIASH_META_MEMORY_PIB+x} ]; then echo "${VIASH_META_MEMORY_PIB}"
## VIASH END
if [[ ! -d "\$par_output" ]]; then
mkdir -p "\$par_output"
fi
extra_params=( )
if [ "\$par_keep_symbolic_links" == "true" ]; then
extra_params+=( "-d" )
# Set copy flags based on options
if [[ "\$par_keep_symbolic_links" == "true" ]]; then
# -a implies -dR --preserve=all
# with -d: same as --no-dereference (and --preserve=links)
# and --no-dereference: never follow symbolic links in SOURCE
# and -R, -r, --recursive: copy directories recursively
# --keep-directory-symlink: if the destination already exists and is a
# symbolic link to a directory, follow the symlink and copy into the directory
# it points to, instead of removing the symlink and creating a real directory in its place.
cp_flags="-a --keep-directory-symlink"
else
# -L: always follow symbolic links in SOURCE
cp_flags="-Lr --preserve=all --no-preserve=link --keep-directory-symlink"
fi
# Process multiple input files
IFS=";" read -ra input_files <<< "\$par_input"
for file in "\${input_files[@]}"; do
# Check if the file exists before copying
if [[ -f "\$file" ]]; then
cp \${extra_params[@]} "\$file" "\$par_output/"
echo "Copied \$file to \$par_output/"
# Process multiple input paths (files or directories)
IFS=";" read -ra input_paths <<< "\$par_input"
for path in "\${input_paths[@]}"; do
if [[ -e "\$path" ]] || [[ -L "\$path" ]]; then
cp \$cp_flags "\$path" "\$par_output/"
echo "Copied \$path to \$par_output/"
else
echo "Warning: Input file \$file does not exist, skipping"
echo "Warning: Input path \$path does not exist, skipping"
fi
done
VIASHMAIN

View File

@@ -196,8 +196,8 @@ build_info:
engine: "docker|native"
output: "target/executable/sync_resources"
executable: "target/executable/sync_resources/sync_resources"
viash_version: "0.9.4"
git_commit: "32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550"
viash_version: "0.9.7"
git_commit: "b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1"
git_remote: "https://github.com/viash-hub/craftbox"
package_config:
name: "craftbox"
@@ -217,7 +217,7 @@ package_config:
\ to ensure dependency management and a consistent, reproducible runtime environment.\n\
\ * Unit tested where applicable to ensure components function as expected.\n"
info: null
viash_version: "0.9.4"
viash_version: "0.9.7"
source: "src"
target: "target"
config_mods:

View File

@@ -2,7 +2,7 @@
# sync_resources main
#
# This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative
# This wrapper script is auto-generated by viash 0.9.7 and is thus a derivative
# work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data
# Intuitive.
#
@@ -455,9 +455,9 @@ RUN rclone config create s3 s3 anonymous=true
RUN rclone config create gs gcs anonymous=true
LABEL org.opencontainers.image.authors="Robrecht Cannoodt, Dries Schaumont"
LABEL org.opencontainers.image.description="Companion container for running component sync_resources"
LABEL org.opencontainers.image.created="2025-09-01T14:32:37Z"
LABEL org.opencontainers.image.created="2026-06-01T12:52:41Z"
LABEL org.opencontainers.image.source="https://github.com/viash-hub/craftbox"
LABEL org.opencontainers.image.revision="32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550"
LABEL org.opencontainers.image.revision="b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1"
LABEL org.opencontainers.image.version="main"
VIASHDOCKER

View File

@@ -181,8 +181,8 @@ build_info:
engine: "docker|native"
output: "target/executable/untar"
executable: "target/executable/untar/untar"
viash_version: "0.9.4"
git_commit: "32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550"
viash_version: "0.9.7"
git_commit: "b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1"
git_remote: "https://github.com/viash-hub/craftbox"
package_config:
name: "craftbox"
@@ -202,7 +202,7 @@ package_config:
\ to ensure dependency management and a consistent, reproducible runtime environment.\n\
\ * Unit tested where applicable to ensure components function as expected.\n"
info: null
viash_version: "0.9.4"
viash_version: "0.9.7"
source: "src"
target: "target"
config_mods:

View File

@@ -2,7 +2,7 @@
# untar main
#
# This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative
# This wrapper script is auto-generated by viash 0.9.7 and is thus a derivative
# work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data
# Intuitive.
#
@@ -455,9 +455,9 @@ RUN apt-get update && \
LABEL org.opencontainers.image.authors="Dries Schaumont, Robrecht Cannoodt"
LABEL org.opencontainers.image.description="Companion container for running component untar"
LABEL org.opencontainers.image.created="2025-09-01T14:32:37Z"
LABEL org.opencontainers.image.created="2026-06-01T12:52:41Z"
LABEL org.opencontainers.image.source="https://github.com/viash-hub/craftbox"
LABEL org.opencontainers.image.revision="32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550"
LABEL org.opencontainers.image.revision="b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1"
LABEL org.opencontainers.image.version="main"
VIASHDOCKER

View File

@@ -156,8 +156,8 @@ build_info:
engine: "docker|native"
output: "target/nextflow/check_disk_space"
executable: "target/nextflow/check_disk_space/main.nf"
viash_version: "0.9.4"
git_commit: "32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550"
viash_version: "0.9.7"
git_commit: "b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1"
git_remote: "https://github.com/viash-hub/craftbox"
package_config:
name: "craftbox"
@@ -177,7 +177,7 @@ package_config:
\ to ensure dependency management and a consistent, reproducible runtime environment.\n\
\ * Unit tested where applicable to ensure components function as expected.\n"
info: null
viash_version: "0.9.4"
viash_version: "0.9.7"
source: "src"
target: "target"
config_mods:

View File

@@ -1,6 +1,6 @@
// check_disk_space main
//
// This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative
// This wrapper script is auto-generated by viash 0.9.7 and is thus a derivative
// work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data
// Intuitive.
//
@@ -1350,47 +1350,42 @@ def readCsv(file_path) {
def splitRegex = java.util.regex.Pattern.compile(''',(?=(?:[^"]*"[^"]*")*[^"]*$)''')
def removeQuote = java.util.regex.Pattern.compile('''"(.*)"''')
def br = java.nio.file.Files.newBufferedReader(inputFile)
java.nio.file.Files.newBufferedReader(inputFile).withCloseable { br ->
def row = 0
def header = null
def line
def row = -1
def header = null
while (br.ready() && header == null) {
def line = br.readLine()
row++
if (!line.startsWith("#")) {
header = splitRegex.split(line, -1).collect{field ->
m = removeQuote.matcher(field)
m.find() ? m.replaceFirst('$1') : field
}
}
}
assert header != null: "CSV file should contain a header"
while (br.ready()) {
def line = br.readLine()
row++
if (line == null) {
br.close()
break
}
if (!line.startsWith("#")) {
def predata = splitRegex.split(line, -1)
def data = predata.collect{field ->
if (field == "") {
return null
}
def m = removeQuote.matcher(field)
if (m.find()) {
return m.replaceFirst('$1')
} else {
return field
while (header == null && (line = br.readLine()) != null) {
if (!line.startsWith("#")) {
header = splitRegex.split(line, -1).collect { field ->
def m = removeQuote.matcher(field)
m.find() ? m.replaceFirst('$1') : field
}
}
assert header.size() == data.size(): "Row $row should contain the same number as fields as the header"
def dataMap = [header, data].transpose().collectEntries().findAll{it.value != null}
output.add(dataMap)
row++
}
assert header != null : "CSV file should contain a header"
while ((line = br.readLine()) != null) {
row++
if (!line.startsWith("#")) {
def predata = splitRegex.split(line, -1)
def data = predata.collect { field ->
if (field == "") {
return null
}
def m = removeQuote.matcher(field)
if (m.find()) {
return m.replaceFirst('$1')
} else {
return field
}
}
assert header.size() == data.size() : "Row $row should contain the same number as fields as the header"
def dataMap = [header, data].transpose().collectEntries().findAll { it.value != null }
output.add(dataMap)
}
}
}
@@ -1706,10 +1701,25 @@ process publishFilesProc {
]
.transpose()
.collectMany{infile, outfile ->
if (infile.toString() != outfile.toString()) {
def infileString = infile.toString()
def outfileString = outfile.toString()
if (infileString != outfileString) {
/* Trailing slashes are removed from both the source and destination arguments.
From source arguments, this is useful when a source argument may have a trailing slash
and specify a symbolic link to a directory. Without removing the slash, cp will dereference
the symbolic link.
See https://www.gnu.org/software/coreutils/manual/html_node/Trailing-slashes.html#Trailing-slashes-1
For the destination path addding a trailing slash is a problem when publishing directories:
it requires the destination directory to exist. This fails because we only create the parent
directories first.
*/
def regexTrailingSlashes = ~/\/+$/
def infileNoTrailingSlash = infileString - regexTrailingSlashes
def outfileNoTrailingSlash = outfileString - regexTrailingSlashes
[
"[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"",
"cp -r '${infile.toString()}' '${outfile.toString()}'"
"[ -d \"\$(dirname '${outfileNoTrailingSlash}')\" ] || mkdir -p \"\$(dirname '${outfileNoTrailingSlash}')\"",
"cp -a '${infileNoTrailingSlash}' '${outfileNoTrailingSlash}'"
]
} else {
// no need to copy if infile is the same as outfile
@@ -3215,8 +3225,8 @@ meta = [
"runner" : "nextflow",
"engine" : "docker|native",
"output" : "target/nextflow/check_disk_space",
"viash_version" : "0.9.4",
"git_commit" : "32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550",
"viash_version" : "0.9.7",
"git_commit" : "b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1",
"git_remote" : "https://github.com/viash-hub/craftbox"
},
"package_config" : {
@@ -3224,7 +3234,7 @@ meta = [
"version" : "main",
"summary" : "A collection of custom-tailored scripts and applied utilities built with Viash.\n",
"description" : "`craftbox` is a curated collection of custom scripts and utilities designed to tackle context-specific tasks.\n\nEmphasizing the Viash principles, `craftbox` components aim for **reusability**, **reproducibility**, and adherence to **best practices**. Key features generally include:\n\n* **Standalone & Nextflow Ready:** Components are built to run directly via the command line or be smoothly integrated into Nextflow workflows.\n* **Custom Implementations:** Contains scripts and tools developed for particular tasks that may not be found in broader collections.\n* **High Quality Standards (promoted by Viash):**\n * Clear documentation for components and their parameters.\n * Full exposure of underlying script/tool arguments for fine-grained control.\n * Containerized (Docker) to ensure dependency management and a consistent, reproducible runtime environment.\n * Unit tested where applicable to ensure components function as expected.\n",
"viash_version" : "0.9.4",
"viash_version" : "0.9.7",
"source" : "src",
"target" : "target",
"config_mods" : [

View File

@@ -26,6 +26,22 @@ authors:
- name: "Data Intuitive"
href: "https://www.data-intuitive.com"
role: "Data Scientist"
- name: "Robrecht Cannoodt"
roles:
- "contributor"
info:
links:
email: "robrecht@data-intuitive.com"
github: "rcannood"
orcid: "0000-0003-3641-729X"
linkedin: "robrechtcannoodt"
organizations:
- name: "Data Intuitive"
href: "https://www.data-intuitive.com"
role: "Data Science Engineer"
- name: "Open Problems"
href: "https://openproblems.bio"
role: "Core Member"
argument_groups:
- name: "Input arguments"
arguments:
@@ -34,7 +50,7 @@ argument_groups:
description: "A list of (gzipped) text files."
info: null
example:
- "input?.txt.gz"
- "input.txt.gz"
must_exist: true
create_parent: true
required: true
@@ -50,7 +66,7 @@ argument_groups:
direction: "input"
- type: "file"
name: "--output"
description: "File to write the output to, optionally gzipped."
description: "File to write the output to, potentially gzipped."
info: null
example:
- "output.txt"
@@ -64,7 +80,7 @@ resources:
- type: "bash_script"
path: "script.sh"
is_executable: true
summary: "Concatenate a number of text files"
summary: "Concatenate multiple (possibly gzipped) text files"
description: "Concatenate a number of text files, handle gzipped text files gracefully\
\ and\noptionally gzip the output text file.\n\nThis component is useful for concatening\
\ fastq files from different lanes, for instance.\n"
@@ -73,9 +89,8 @@ test_resources:
path: "test.sh"
is_executable: true
info:
improvements: "This component could be improved in 2 ways:\n 1. Allow for a mix\
\ of zipped and plain input files\n 2. Allow to specify a compression algorithm\
\ for the output\n"
improvements: "This component could be improved:\n 1. Allow to specify a compression\
\ algorithm for the output\n"
status: "enabled"
scope:
image: "public"
@@ -174,8 +189,8 @@ build_info:
engine: "docker|native"
output: "target/nextflow/concat_text"
executable: "target/nextflow/concat_text/main.nf"
viash_version: "0.9.4"
git_commit: "32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550"
viash_version: "0.9.7"
git_commit: "b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1"
git_remote: "https://github.com/viash-hub/craftbox"
package_config:
name: "craftbox"
@@ -195,7 +210,7 @@ package_config:
\ to ensure dependency management and a consistent, reproducible runtime environment.\n\
\ * Unit tested where applicable to ensure components function as expected.\n"
info: null
viash_version: "0.9.4"
viash_version: "0.9.7"
source: "src"
target: "target"
config_mods:

View File

@@ -1,6 +1,6 @@
// concat_text main
//
// This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative
// This wrapper script is auto-generated by viash 0.9.7 and is thus a derivative
// work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data
// Intuitive.
//
@@ -12,6 +12,7 @@
// Component authors:
// * Toni Verbeiren (author, maintainer)
// * Dries Schaumont (reviewer)
// * Robrecht Cannoodt (contributor)
////////////////////////////
// VDSL3 helper functions //
@@ -1354,47 +1355,42 @@ def readCsv(file_path) {
def splitRegex = java.util.regex.Pattern.compile(''',(?=(?:[^"]*"[^"]*")*[^"]*$)''')
def removeQuote = java.util.regex.Pattern.compile('''"(.*)"''')
def br = java.nio.file.Files.newBufferedReader(inputFile)
java.nio.file.Files.newBufferedReader(inputFile).withCloseable { br ->
def row = 0
def header = null
def line
def row = -1
def header = null
while (br.ready() && header == null) {
def line = br.readLine()
row++
if (!line.startsWith("#")) {
header = splitRegex.split(line, -1).collect{field ->
m = removeQuote.matcher(field)
m.find() ? m.replaceFirst('$1') : field
}
}
}
assert header != null: "CSV file should contain a header"
while (br.ready()) {
def line = br.readLine()
row++
if (line == null) {
br.close()
break
}
if (!line.startsWith("#")) {
def predata = splitRegex.split(line, -1)
def data = predata.collect{field ->
if (field == "") {
return null
}
def m = removeQuote.matcher(field)
if (m.find()) {
return m.replaceFirst('$1')
} else {
return field
while (header == null && (line = br.readLine()) != null) {
if (!line.startsWith("#")) {
header = splitRegex.split(line, -1).collect { field ->
def m = removeQuote.matcher(field)
m.find() ? m.replaceFirst('$1') : field
}
}
assert header.size() == data.size(): "Row $row should contain the same number as fields as the header"
def dataMap = [header, data].transpose().collectEntries().findAll{it.value != null}
output.add(dataMap)
row++
}
assert header != null : "CSV file should contain a header"
while ((line = br.readLine()) != null) {
row++
if (!line.startsWith("#")) {
def predata = splitRegex.split(line, -1)
def data = predata.collect { field ->
if (field == "") {
return null
}
def m = removeQuote.matcher(field)
if (m.find()) {
return m.replaceFirst('$1')
} else {
return field
}
}
assert header.size() == data.size() : "Row $row should contain the same number as fields as the header"
def dataMap = [header, data].transpose().collectEntries().findAll { it.value != null }
output.add(dataMap)
}
}
}
@@ -1710,10 +1706,25 @@ process publishFilesProc {
]
.transpose()
.collectMany{infile, outfile ->
if (infile.toString() != outfile.toString()) {
def infileString = infile.toString()
def outfileString = outfile.toString()
if (infileString != outfileString) {
/* Trailing slashes are removed from both the source and destination arguments.
From source arguments, this is useful when a source argument may have a trailing slash
and specify a symbolic link to a directory. Without removing the slash, cp will dereference
the symbolic link.
See https://www.gnu.org/software/coreutils/manual/html_node/Trailing-slashes.html#Trailing-slashes-1
For the destination path addding a trailing slash is a problem when publishing directories:
it requires the destination directory to exist. This fails because we only create the parent
directories first.
*/
def regexTrailingSlashes = ~/\/+$/
def infileNoTrailingSlash = infileString - regexTrailingSlashes
def outfileNoTrailingSlash = outfileString - regexTrailingSlashes
[
"[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"",
"cp -r '${infile.toString()}' '${outfile.toString()}'"
"[ -d \"\$(dirname '${outfileNoTrailingSlash}')\" ] || mkdir -p \"\$(dirname '${outfileNoTrailingSlash}')\"",
"cp -a '${infileNoTrailingSlash}' '${outfileNoTrailingSlash}'"
]
} else {
// no need to copy if infile is the same as outfile
@@ -3077,6 +3088,32 @@ meta = [
}
]
}
},
{
"name" : "Robrecht Cannoodt",
"roles" : [
"contributor"
],
"info" : {
"links" : {
"email" : "robrecht@data-intuitive.com",
"github" : "rcannood",
"orcid" : "0000-0003-3641-729X",
"linkedin" : "robrechtcannoodt"
},
"organizations" : [
{
"name" : "Data Intuitive",
"href" : "https://www.data-intuitive.com",
"role" : "Data Science Engineer"
},
{
"name" : "Open Problems",
"href" : "https://openproblems.bio",
"role" : "Core Member"
}
]
}
}
],
"argument_groups" : [
@@ -3088,7 +3125,7 @@ meta = [
"name" : "--input",
"description" : "A list of (gzipped) text files.",
"example" : [
"input?.txt.gz"
"input.txt.gz"
],
"must_exist" : true,
"create_parent" : true,
@@ -3111,7 +3148,7 @@ meta = [
{
"type" : "file",
"name" : "--output",
"description" : "File to write the output to, optionally gzipped.",
"description" : "File to write the output to, potentially gzipped.",
"example" : [
"output.txt"
],
@@ -3132,7 +3169,7 @@ meta = [
"is_executable" : true
}
],
"summary" : "Concatenate a number of text files",
"summary" : "Concatenate multiple (possibly gzipped) text files",
"description" : "Concatenate a number of text files, handle gzipped text files gracefully and\noptionally gzip the output text file.\n\nThis component is useful for concatening fastq files from different lanes, for instance.\n",
"test_resources" : [
{
@@ -3142,7 +3179,7 @@ meta = [
}
],
"info" : {
"improvements" : "This component could be improved in 2 ways:\n 1. Allow for a mix of zipped and plain input files\n 2. Allow to specify a compression algorithm for the output\n"
"improvements" : "This component could be improved:\n 1. Allow to specify a compression algorithm for the output\n"
},
"status" : "enabled",
"scope" : {
@@ -3261,8 +3298,8 @@ meta = [
"runner" : "nextflow",
"engine" : "docker|native",
"output" : "target/nextflow/concat_text",
"viash_version" : "0.9.4",
"git_commit" : "32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550",
"viash_version" : "0.9.7",
"git_commit" : "b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1",
"git_remote" : "https://github.com/viash-hub/craftbox"
},
"package_config" : {
@@ -3270,7 +3307,7 @@ meta = [
"version" : "main",
"summary" : "A collection of custom-tailored scripts and applied utilities built with Viash.\n",
"description" : "`craftbox` is a curated collection of custom scripts and utilities designed to tackle context-specific tasks.\n\nEmphasizing the Viash principles, `craftbox` components aim for **reusability**, **reproducibility**, and adherence to **best practices**. Key features generally include:\n\n* **Standalone & Nextflow Ready:** Components are built to run directly via the command line or be smoothly integrated into Nextflow workflows.\n* **Custom Implementations:** Contains scripts and tools developed for particular tasks that may not be found in broader collections.\n* **High Quality Standards (promoted by Viash):**\n * Clear documentation for components and their parameters.\n * Full exposure of underlying script/tool arguments for fine-grained control.\n * Containerized (Docker) to ensure dependency management and a consistent, reproducible runtime environment.\n * Unit tested where applicable to ensure components function as expected.\n",
"viash_version" : "0.9.4",
"viash_version" : "0.9.7",
"source" : "src",
"target" : "target",
"config_mods" : [
@@ -3304,6 +3341,10 @@ def innerWorkflowFactory(args) {
def rawScript = '''set -e
tempscript=".viash_script.sh"
cat > "$tempscript" << VIASHMAIN
#!/usr/bin/env bash
set -eo pipefail
## VIASH START
# The following code has been auto-generated by Viash.
$( if [ ! -z ${VIASH_PAR_INPUT+x} ]; then echo "${VIASH_PAR_INPUT}" | sed "s#'#'\\"'\\"'#g;s#.*#par_input='&'#" ; else echo "# par_input="; fi )
@@ -3329,40 +3370,62 @@ $( if [ ! -z ${VIASH_META_MEMORY_TIB+x} ]; then echo "${VIASH_META_MEMORY_TIB}"
$( if [ ! -z ${VIASH_META_MEMORY_PIB+x} ]; then echo "${VIASH_META_MEMORY_PIB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_pib='&'#" ; else echo "# meta_memory_pib="; fi )
## VIASH END
#!/usr/bin/env bash
set -euo pipefail
TMPDIR=\\$(mktemp -d "\\$meta_temp_dir/concat_text-XXXXXX")
function clean_up {
[[ -d "\\$TMPDIR" ]] && rm -r "\\$TMPDIR"
# --- Function to check for GZIP format using the 'file' command ---
is_gzipped() {
# Ensure the file exists and is not empty before checking
if [ ! -s "\\$1" ]; then
return 1
fi
# Get the MIME type of the file. The '-b' option omits the filename from the output.
local mime_type
mime_type=\\$(file -b --mime-type "\\$1")
# Check if the MIME type corresponds to gzip.
# application/gzip is standard, while application/x-gzip is also commonly seen.
if [[ "\\$mime_type" == "application/gzip" || "\\$mime_type" == "application/x-gzip" ]]; then
return 0 # 0 indicates success (true in bash)
else
return 1 # 1 indicates failure (false in bash)
fi
}
trap clean_up EXIT
par_input="\\$(echo "\\$par_input" | tr ';' ' ')"
# Read the ;-separated file paths from the input variable into an array
IFS=";" read -ra input_files <<< "\\$par_input"
echo -n ">> Check if input is gzipped... "
set +eo pipefail
file \\$par_input | grep -q 'gzip'
is_zipped="\\$?"
set -euo pipefail
[[ "\\$is_zipped" == "0" ]] && echo "yes" || echo "no"
# Process the files if the array contains any paths
if [ \\${#input_files[@]} -gt 0 ] && [ -n "\\${input_files[0]}" ]; then
if [[ "\\$is_zipped" == "0" ]]; then
echo ">> zcat gzipped files"
zcat \\$par_input > \\$TMPDIR/contents
# Ensure the output file is empty before we start
> "\\$par_output"
echo "Processing files for -> \\$par_output"
# Create a subshell for the loop to group all cat/zcat output.
(
for file in "\\${input_files[@]}"; do
if [ -z "\\$file" ]; then continue; fi # Skip empty entries in the array
if is_gzipped "\\$file"; then
zcat "\\$file"
else
cat "\\$file"
fi
done
) | if [ "\\$par_compress_output" = "true" ]; then
# If compression is enabled, pipe the entire stream to gzip
gzip -c >> "\\$par_output"
else
# Otherwise, just redirect the stream to the plain text file
cat >> "\\$par_output"
fi
echo "Finished creating \\$par_output."
else
echo ">> cat plain files"
cat \\$par_input > \\$TMPDIR/contents
echo "No input files provided in \\\\\\$par_input. Exiting."
fi
if [ "\\$par_gzip_output" == true ]; then
echo ">> Zip output file"
gzip \\$TMPDIR/contents
mv \\$TMPDIR/contents.gz \\$par_output
else
mv \\$TMPDIR/contents \\$par_output
fi
echo "Script finished successfully."
VIASHMAIN
bash "$tempscript"
'''

View File

@@ -4,7 +4,7 @@ manifest {
nextflowVersion = '!>=20.12.1-edge'
version = 'main'
description = 'Concatenate a number of text files, handle gzipped text files gracefully and\noptionally gzip the output text file.\n\nThis component is useful for concatening fastq files from different lanes, for instance.\n'
author = 'Toni Verbeiren, Dries Schaumont'
author = 'Toni Verbeiren, Dries Schaumont, Robrecht Cannoodt'
}
process.container = 'nextflow/bash:latest'

View File

@@ -17,7 +17,7 @@
"format": "path",
"exists": true,
"description": "A list of (gzipped) text files.",
"help_text": "Type: `file`, multiple: `True`, required, direction: `input`, example: `[\"input?.txt.gz\"]`. "
"help_text": "Type: `file`, multiple: `True`, required, direction: `input`, example: `[\"input.txt.gz\"]`. "
}
}
},
@@ -35,7 +35,7 @@
"output": {
"type": "string",
"format": "path",
"description": "File to write the output to, optionally gzipped.",
"description": "File to write the output to, potentially gzipped.",
"help_text": "Type: `file`, multiple: `False`, default: `\"$id.$key.output.txt\"`, direction: `output`, example: `\"output.txt\"`. ",
"default": "$id.$key.output.txt"
}

View File

@@ -256,8 +256,8 @@ build_info:
engine: "docker|native"
output: "target/nextflow/csv2fasta"
executable: "target/nextflow/csv2fasta/main.nf"
viash_version: "0.9.4"
git_commit: "32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550"
viash_version: "0.9.7"
git_commit: "b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1"
git_remote: "https://github.com/viash-hub/craftbox"
package_config:
name: "craftbox"
@@ -277,7 +277,7 @@ package_config:
\ to ensure dependency management and a consistent, reproducible runtime environment.\n\
\ * Unit tested where applicable to ensure components function as expected.\n"
info: null
viash_version: "0.9.4"
viash_version: "0.9.7"
source: "src"
target: "target"
config_mods:

View File

@@ -1,6 +1,6 @@
// csv2fasta main
//
// This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative
// This wrapper script is auto-generated by viash 0.9.7 and is thus a derivative
// work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data
// Intuitive.
//
@@ -1354,47 +1354,42 @@ def readCsv(file_path) {
def splitRegex = java.util.regex.Pattern.compile(''',(?=(?:[^"]*"[^"]*")*[^"]*$)''')
def removeQuote = java.util.regex.Pattern.compile('''"(.*)"''')
def br = java.nio.file.Files.newBufferedReader(inputFile)
java.nio.file.Files.newBufferedReader(inputFile).withCloseable { br ->
def row = 0
def header = null
def line
def row = -1
def header = null
while (br.ready() && header == null) {
def line = br.readLine()
row++
if (!line.startsWith("#")) {
header = splitRegex.split(line, -1).collect{field ->
m = removeQuote.matcher(field)
m.find() ? m.replaceFirst('$1') : field
}
}
}
assert header != null: "CSV file should contain a header"
while (br.ready()) {
def line = br.readLine()
row++
if (line == null) {
br.close()
break
}
if (!line.startsWith("#")) {
def predata = splitRegex.split(line, -1)
def data = predata.collect{field ->
if (field == "") {
return null
}
def m = removeQuote.matcher(field)
if (m.find()) {
return m.replaceFirst('$1')
} else {
return field
while (header == null && (line = br.readLine()) != null) {
if (!line.startsWith("#")) {
header = splitRegex.split(line, -1).collect { field ->
def m = removeQuote.matcher(field)
m.find() ? m.replaceFirst('$1') : field
}
}
assert header.size() == data.size(): "Row $row should contain the same number as fields as the header"
def dataMap = [header, data].transpose().collectEntries().findAll{it.value != null}
output.add(dataMap)
row++
}
assert header != null : "CSV file should contain a header"
while ((line = br.readLine()) != null) {
row++
if (!line.startsWith("#")) {
def predata = splitRegex.split(line, -1)
def data = predata.collect { field ->
if (field == "") {
return null
}
def m = removeQuote.matcher(field)
if (m.find()) {
return m.replaceFirst('$1')
} else {
return field
}
}
assert header.size() == data.size() : "Row $row should contain the same number as fields as the header"
def dataMap = [header, data].transpose().collectEntries().findAll { it.value != null }
output.add(dataMap)
}
}
}
@@ -1710,10 +1705,25 @@ process publishFilesProc {
]
.transpose()
.collectMany{infile, outfile ->
if (infile.toString() != outfile.toString()) {
def infileString = infile.toString()
def outfileString = outfile.toString()
if (infileString != outfileString) {
/* Trailing slashes are removed from both the source and destination arguments.
From source arguments, this is useful when a source argument may have a trailing slash
and specify a symbolic link to a directory. Without removing the slash, cp will dereference
the symbolic link.
See https://www.gnu.org/software/coreutils/manual/html_node/Trailing-slashes.html#Trailing-slashes-1
For the destination path addding a trailing slash is a problem when publishing directories:
it requires the destination directory to exist. This fails because we only create the parent
directories first.
*/
def regexTrailingSlashes = ~/\/+$/
def infileNoTrailingSlash = infileString - regexTrailingSlashes
def outfileNoTrailingSlash = outfileString - regexTrailingSlashes
[
"[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"",
"cp -r '${infile.toString()}' '${outfile.toString()}'"
"[ -d \"\$(dirname '${outfileNoTrailingSlash}')\" ] || mkdir -p \"\$(dirname '${outfileNoTrailingSlash}')\"",
"cp -a '${infileNoTrailingSlash}' '${outfileNoTrailingSlash}'"
]
} else {
// no need to copy if infile is the same as outfile
@@ -3351,8 +3361,8 @@ meta = [
"runner" : "nextflow",
"engine" : "docker|native",
"output" : "target/nextflow/csv2fasta",
"viash_version" : "0.9.4",
"git_commit" : "32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550",
"viash_version" : "0.9.7",
"git_commit" : "b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1",
"git_remote" : "https://github.com/viash-hub/craftbox"
},
"package_config" : {
@@ -3360,7 +3370,7 @@ meta = [
"version" : "main",
"summary" : "A collection of custom-tailored scripts and applied utilities built with Viash.\n",
"description" : "`craftbox` is a curated collection of custom scripts and utilities designed to tackle context-specific tasks.\n\nEmphasizing the Viash principles, `craftbox` components aim for **reusability**, **reproducibility**, and adherence to **best practices**. Key features generally include:\n\n* **Standalone & Nextflow Ready:** Components are built to run directly via the command line or be smoothly integrated into Nextflow workflows.\n* **Custom Implementations:** Contains scripts and tools developed for particular tasks that may not be found in broader collections.\n* **High Quality Standards (promoted by Viash):**\n * Clear documentation for components and their parameters.\n * Full exposure of underlying script/tool arguments for fine-grained control.\n * Containerized (Docker) to ensure dependency management and a consistent, reproducible runtime environment.\n * Unit tested where applicable to ensure components function as expected.\n",
"viash_version" : "0.9.4",
"viash_version" : "0.9.7",
"source" : "src",
"target" : "target",
"config_mods" : [

View File

@@ -14,11 +14,12 @@ authors:
href: "https://www.data-intuitive.com"
role: "Data Scientist"
argument_groups:
- name: "Arguments"
- name: "Inputs"
arguments:
- type: "file"
name: "--input"
description: "Paths of the files that will be copied into the output directory."
description: "Paths of the files or directories that will be copied into the output\
\ directory."
info: null
must_exist: true
create_parent: true
@@ -26,9 +27,11 @@ argument_groups:
direction: "input"
multiple: true
multiple_sep: ";"
- name: "Outputs"
arguments:
- type: "file"
name: "--output"
description: "Path to output directory"
description: "Path to output directory."
info: null
must_exist: true
create_parent: true
@@ -36,20 +39,22 @@ argument_groups:
direction: "output"
multiple: false
multiple_sep: ";"
- name: "Options"
arguments:
- type: "boolean_true"
name: "--keep_symbolic_links"
alternatives:
- "-d"
description: "Preserve symbolic links."
description: "When set, symbolic links are preserved as symbolic links in the\
\ output directory. By default, symbolic links are dereferenced and the target\
\ file is copied."
info: null
direction: "input"
resources:
- type: "bash_script"
path: "script.sh"
is_executable: true
summary: "Publish one or multiple files to the same directory"
description: "This component copies one or multiple files to the same destination\
\ directory, creating the output directory if it doesn't exist."
summary: "Publish one or multiple files or directories to the same output directory."
description: "This component copies one or multiple files or directories\nto the same\
\ destination directory, creating the output directory if it doesn't\nexist.\n"
test_resources:
- type: "bash_script"
path: "test.sh"
@@ -146,14 +151,16 @@ engines:
cmd: null
- type: "native"
id: "native"
- type: "native"
id: "native"
build_info:
config: "src/move_files_to_directory/config.vsh.yaml"
runner: "nextflow"
engine: "docker|native"
engine: "docker|native|native"
output: "target/nextflow/move_files_to_directory"
executable: "target/nextflow/move_files_to_directory/main.nf"
viash_version: "0.9.4"
git_commit: "32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550"
viash_version: "0.9.7"
git_commit: "b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1"
git_remote: "https://github.com/viash-hub/craftbox"
package_config:
name: "craftbox"
@@ -173,7 +180,7 @@ package_config:
\ to ensure dependency management and a consistent, reproducible runtime environment.\n\
\ * Unit tested where applicable to ensure components function as expected.\n"
info: null
viash_version: "0.9.4"
viash_version: "0.9.7"
source: "src"
target: "target"
config_mods:

View File

@@ -1,6 +1,6 @@
// move_files_to_directory main
//
// This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative
// This wrapper script is auto-generated by viash 0.9.7 and is thus a derivative
// work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data
// Intuitive.
//
@@ -1353,47 +1353,42 @@ def readCsv(file_path) {
def splitRegex = java.util.regex.Pattern.compile(''',(?=(?:[^"]*"[^"]*")*[^"]*$)''')
def removeQuote = java.util.regex.Pattern.compile('''"(.*)"''')
def br = java.nio.file.Files.newBufferedReader(inputFile)
java.nio.file.Files.newBufferedReader(inputFile).withCloseable { br ->
def row = 0
def header = null
def line
def row = -1
def header = null
while (br.ready() && header == null) {
def line = br.readLine()
row++
if (!line.startsWith("#")) {
header = splitRegex.split(line, -1).collect{field ->
m = removeQuote.matcher(field)
m.find() ? m.replaceFirst('$1') : field
}
}
}
assert header != null: "CSV file should contain a header"
while (br.ready()) {
def line = br.readLine()
row++
if (line == null) {
br.close()
break
}
if (!line.startsWith("#")) {
def predata = splitRegex.split(line, -1)
def data = predata.collect{field ->
if (field == "") {
return null
}
def m = removeQuote.matcher(field)
if (m.find()) {
return m.replaceFirst('$1')
} else {
return field
while (header == null && (line = br.readLine()) != null) {
if (!line.startsWith("#")) {
header = splitRegex.split(line, -1).collect { field ->
def m = removeQuote.matcher(field)
m.find() ? m.replaceFirst('$1') : field
}
}
assert header.size() == data.size(): "Row $row should contain the same number as fields as the header"
def dataMap = [header, data].transpose().collectEntries().findAll{it.value != null}
output.add(dataMap)
row++
}
assert header != null : "CSV file should contain a header"
while ((line = br.readLine()) != null) {
row++
if (!line.startsWith("#")) {
def predata = splitRegex.split(line, -1)
def data = predata.collect { field ->
if (field == "") {
return null
}
def m = removeQuote.matcher(field)
if (m.find()) {
return m.replaceFirst('$1')
} else {
return field
}
}
assert header.size() == data.size() : "Row $row should contain the same number as fields as the header"
def dataMap = [header, data].transpose().collectEntries().findAll { it.value != null }
output.add(dataMap)
}
}
}
@@ -1709,10 +1704,25 @@ process publishFilesProc {
]
.transpose()
.collectMany{infile, outfile ->
if (infile.toString() != outfile.toString()) {
def infileString = infile.toString()
def outfileString = outfile.toString()
if (infileString != outfileString) {
/* Trailing slashes are removed from both the source and destination arguments.
From source arguments, this is useful when a source argument may have a trailing slash
and specify a symbolic link to a directory. Without removing the slash, cp will dereference
the symbolic link.
See https://www.gnu.org/software/coreutils/manual/html_node/Trailing-slashes.html#Trailing-slashes-1
For the destination path addding a trailing slash is a problem when publishing directories:
it requires the destination directory to exist. This fails because we only create the parent
directories first.
*/
def regexTrailingSlashes = ~/\/+$/
def infileNoTrailingSlash = infileString - regexTrailingSlashes
def outfileNoTrailingSlash = outfileString - regexTrailingSlashes
[
"[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"",
"cp -r '${infile.toString()}' '${outfile.toString()}'"
"[ -d \"\$(dirname '${outfileNoTrailingSlash}')\" ] || mkdir -p \"\$(dirname '${outfileNoTrailingSlash}')\"",
"cp -a '${infileNoTrailingSlash}' '${outfileNoTrailingSlash}'"
]
} else {
// no need to copy if infile is the same as outfile
@@ -3059,37 +3069,44 @@ meta = [
],
"argument_groups" : [
{
"name" : "Arguments",
"name" : "Inputs",
"arguments" : [
{
"type" : "file",
"name" : "--input",
"description" : "Paths of the files that will be copied into the output directory.",
"description" : "Paths of the files or directories that will be copied into the output directory.",
"must_exist" : true,
"create_parent" : true,
"required" : true,
"direction" : "input",
"multiple" : true,
"multiple_sep" : ";"
},
}
]
},
{
"name" : "Outputs",
"arguments" : [
{
"type" : "file",
"name" : "--output",
"description" : "Path to output directory",
"description" : "Path to output directory.",
"must_exist" : true,
"create_parent" : true,
"required" : true,
"direction" : "output",
"multiple" : false,
"multiple_sep" : ";"
},
}
]
},
{
"name" : "Options",
"arguments" : [
{
"type" : "boolean_true",
"name" : "--keep_symbolic_links",
"alternatives" : [
"-d"
],
"description" : "Preserve symbolic links.",
"description" : "When set, symbolic links are preserved as symbolic links in the output directory. By default, symbolic links are dereferenced and the target file is copied.",
"direction" : "input"
}
]
@@ -3102,8 +3119,8 @@ meta = [
"is_executable" : true
}
],
"summary" : "Publish one or multiple files to the same directory",
"description" : "This component copies one or multiple files to the same destination directory, creating the output directory if it doesn't exist.",
"summary" : "Publish one or multiple files or directories to the same output directory.",
"description" : "This component copies one or multiple files or directories\nto the same destination directory, creating the output directory if it doesn't\nexist.\n",
"test_resources" : [
{
"type" : "bash_script",
@@ -3217,6 +3234,10 @@ meta = [
}
]
},
{
"type" : "native",
"id" : "native"
},
{
"type" : "native",
"id" : "native"
@@ -3225,10 +3246,10 @@ meta = [
"build_info" : {
"config" : "/workdir/root/repo/src/move_files_to_directory/config.vsh.yaml",
"runner" : "nextflow",
"engine" : "docker|native",
"engine" : "docker|native|native",
"output" : "target/nextflow/move_files_to_directory",
"viash_version" : "0.9.4",
"git_commit" : "32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550",
"viash_version" : "0.9.7",
"git_commit" : "b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1",
"git_remote" : "https://github.com/viash-hub/craftbox"
},
"package_config" : {
@@ -3236,7 +3257,7 @@ meta = [
"version" : "main",
"summary" : "A collection of custom-tailored scripts and applied utilities built with Viash.\n",
"description" : "`craftbox` is a curated collection of custom scripts and utilities designed to tackle context-specific tasks.\n\nEmphasizing the Viash principles, `craftbox` components aim for **reusability**, **reproducibility**, and adherence to **best practices**. Key features generally include:\n\n* **Standalone & Nextflow Ready:** Components are built to run directly via the command line or be smoothly integrated into Nextflow workflows.\n* **Custom Implementations:** Contains scripts and tools developed for particular tasks that may not be found in broader collections.\n* **High Quality Standards (promoted by Viash):**\n * Clear documentation for components and their parameters.\n * Full exposure of underlying script/tool arguments for fine-grained control.\n * Containerized (Docker) to ensure dependency management and a consistent, reproducible runtime environment.\n * Unit tested where applicable to ensure components function as expected.\n",
"viash_version" : "0.9.4",
"viash_version" : "0.9.7",
"source" : "src",
"target" : "target",
"config_mods" : [
@@ -3300,27 +3321,33 @@ $( if [ ! -z ${VIASH_META_MEMORY_PIB+x} ]; then echo "${VIASH_META_MEMORY_PIB}"
## VIASH END
if [[ ! -d "\\$par_output" ]]; then
mkdir -p "\\$par_output"
fi
extra_params=( )
if [ "\\$par_keep_symbolic_links" == "true" ]; then
extra_params+=( "-d" )
# Set copy flags based on options
if [[ "\\$par_keep_symbolic_links" == "true" ]]; then
# -a implies -dR --preserve=all
# with -d: same as --no-dereference (and --preserve=links)
# and --no-dereference: never follow symbolic links in SOURCE
# and -R, -r, --recursive: copy directories recursively
# --keep-directory-symlink: if the destination already exists and is a
# symbolic link to a directory, follow the symlink and copy into the directory
# it points to, instead of removing the symlink and creating a real directory in its place.
cp_flags="-a --keep-directory-symlink"
else
# -L: always follow symbolic links in SOURCE
cp_flags="-Lr --preserve=all --no-preserve=link --keep-directory-symlink"
fi
# Process multiple input files
IFS=";" read -ra input_files <<< "\\$par_input"
for file in "\\${input_files[@]}"; do
# Check if the file exists before copying
if [[ -f "\\$file" ]]; then
cp \\${extra_params[@]} "\\$file" "\\$par_output/"
echo "Copied \\$file to \\$par_output/"
# Process multiple input paths (files or directories)
IFS=";" read -ra input_paths <<< "\\$par_input"
for path in "\\${input_paths[@]}"; do
if [[ -e "\\$path" ]] || [[ -L "\\$path" ]]; then
cp \\$cp_flags "\\$path" "\\$par_output/"
echo "Copied \\$path to \\$par_output/"
else
echo "Warning: Input file \\$file does not exist, skipping"
echo "Warning: Input path \\$path does not exist, skipping"
fi
done
VIASHMAIN

View File

@@ -3,7 +3,7 @@ manifest {
mainScript = 'main.nf'
nextflowVersion = '!>=20.12.1-edge'
version = 'main'
description = 'This component copies one or multiple files to the same destination directory, creating the output directory if it doesn\'t exist.'
description = 'This component copies one or multiple files or directories\nto the same destination directory, creating the output directory if it doesn\'t\nexist.\n'
author = 'Dorien Roosen'
}

View File

@@ -1,11 +1,11 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "move_files_to_directory",
"description": "This component copies one or multiple files to the same destination directory, creating the output directory if it doesn't exist.",
"description": "This component copies one or multiple files or directories\nto the same destination directory, creating the output directory if it doesn't\nexist.\n",
"type": "object",
"$defs": {
"arguments": {
"title": "Arguments",
"inputs": {
"title": "Inputs",
"type": "object",
"description": "No description",
"properties": {
@@ -16,19 +16,33 @@
},
"format": "path",
"exists": true,
"description": "Paths of the files that will be copied into the output directory.",
"description": "Paths of the files or directories that will be copied into the output directory.",
"help_text": "Type: `file`, multiple: `True`, required, direction: `input`. "
},
}
}
},
"outputs": {
"title": "Outputs",
"type": "object",
"description": "No description",
"properties": {
"output": {
"type": "string",
"format": "path",
"description": "Path to output directory",
"description": "Path to output directory.",
"help_text": "Type: `file`, multiple: `False`, required, default: `\"$id.$key.output\"`, direction: `output`. ",
"default": "$id.$key.output"
},
}
}
},
"options": {
"title": "Options",
"type": "object",
"description": "No description",
"properties": {
"keep_symbolic_links": {
"type": "boolean",
"description": "Preserve symbolic links.",
"description": "When set, symbolic links are preserved as symbolic links in the output directory",
"help_text": "Type: `boolean_true`, multiple: `False`, default: `false`. ",
"default": false
}
@@ -49,7 +63,13 @@
},
"allOf": [
{
"$ref": "#/$defs/arguments"
"$ref": "#/$defs/inputs"
},
{
"$ref": "#/$defs/outputs"
},
{
"$ref": "#/$defs/options"
},
{
"$ref": "#/$defs/nextflow input-output arguments"

View File

@@ -196,8 +196,8 @@ build_info:
engine: "docker|native"
output: "target/nextflow/sync_resources"
executable: "target/nextflow/sync_resources/main.nf"
viash_version: "0.9.4"
git_commit: "32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550"
viash_version: "0.9.7"
git_commit: "b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1"
git_remote: "https://github.com/viash-hub/craftbox"
package_config:
name: "craftbox"
@@ -217,7 +217,7 @@ package_config:
\ to ensure dependency management and a consistent, reproducible runtime environment.\n\
\ * Unit tested where applicable to ensure components function as expected.\n"
info: null
viash_version: "0.9.4"
viash_version: "0.9.7"
source: "src"
target: "target"
config_mods:

View File

@@ -1,6 +1,6 @@
// sync_resources main
//
// This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative
// This wrapper script is auto-generated by viash 0.9.7 and is thus a derivative
// work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data
// Intuitive.
//
@@ -1354,47 +1354,42 @@ def readCsv(file_path) {
def splitRegex = java.util.regex.Pattern.compile(''',(?=(?:[^"]*"[^"]*")*[^"]*$)''')
def removeQuote = java.util.regex.Pattern.compile('''"(.*)"''')
def br = java.nio.file.Files.newBufferedReader(inputFile)
java.nio.file.Files.newBufferedReader(inputFile).withCloseable { br ->
def row = 0
def header = null
def line
def row = -1
def header = null
while (br.ready() && header == null) {
def line = br.readLine()
row++
if (!line.startsWith("#")) {
header = splitRegex.split(line, -1).collect{field ->
m = removeQuote.matcher(field)
m.find() ? m.replaceFirst('$1') : field
}
}
}
assert header != null: "CSV file should contain a header"
while (br.ready()) {
def line = br.readLine()
row++
if (line == null) {
br.close()
break
}
if (!line.startsWith("#")) {
def predata = splitRegex.split(line, -1)
def data = predata.collect{field ->
if (field == "") {
return null
}
def m = removeQuote.matcher(field)
if (m.find()) {
return m.replaceFirst('$1')
} else {
return field
while (header == null && (line = br.readLine()) != null) {
if (!line.startsWith("#")) {
header = splitRegex.split(line, -1).collect { field ->
def m = removeQuote.matcher(field)
m.find() ? m.replaceFirst('$1') : field
}
}
assert header.size() == data.size(): "Row $row should contain the same number as fields as the header"
def dataMap = [header, data].transpose().collectEntries().findAll{it.value != null}
output.add(dataMap)
row++
}
assert header != null : "CSV file should contain a header"
while ((line = br.readLine()) != null) {
row++
if (!line.startsWith("#")) {
def predata = splitRegex.split(line, -1)
def data = predata.collect { field ->
if (field == "") {
return null
}
def m = removeQuote.matcher(field)
if (m.find()) {
return m.replaceFirst('$1')
} else {
return field
}
}
assert header.size() == data.size() : "Row $row should contain the same number as fields as the header"
def dataMap = [header, data].transpose().collectEntries().findAll { it.value != null }
output.add(dataMap)
}
}
}
@@ -1710,10 +1705,25 @@ process publishFilesProc {
]
.transpose()
.collectMany{infile, outfile ->
if (infile.toString() != outfile.toString()) {
def infileString = infile.toString()
def outfileString = outfile.toString()
if (infileString != outfileString) {
/* Trailing slashes are removed from both the source and destination arguments.
From source arguments, this is useful when a source argument may have a trailing slash
and specify a symbolic link to a directory. Without removing the slash, cp will dereference
the symbolic link.
See https://www.gnu.org/software/coreutils/manual/html_node/Trailing-slashes.html#Trailing-slashes-1
For the destination path addding a trailing slash is a problem when publishing directories:
it requires the destination directory to exist. This fails because we only create the parent
directories first.
*/
def regexTrailingSlashes = ~/\/+$/
def infileNoTrailingSlash = infileString - regexTrailingSlashes
def outfileNoTrailingSlash = outfileString - regexTrailingSlashes
[
"[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"",
"cp -r '${infile.toString()}' '${outfile.toString()}'"
"[ -d \"\$(dirname '${outfileNoTrailingSlash}')\" ] || mkdir -p \"\$(dirname '${outfileNoTrailingSlash}')\"",
"cp -a '${infileNoTrailingSlash}' '${outfileNoTrailingSlash}'"
]
} else {
// no need to copy if infile is the same as outfile
@@ -3293,8 +3303,8 @@ meta = [
"runner" : "nextflow",
"engine" : "docker|native",
"output" : "target/nextflow/sync_resources",
"viash_version" : "0.9.4",
"git_commit" : "32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550",
"viash_version" : "0.9.7",
"git_commit" : "b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1",
"git_remote" : "https://github.com/viash-hub/craftbox"
},
"package_config" : {
@@ -3302,7 +3312,7 @@ meta = [
"version" : "main",
"summary" : "A collection of custom-tailored scripts and applied utilities built with Viash.\n",
"description" : "`craftbox` is a curated collection of custom scripts and utilities designed to tackle context-specific tasks.\n\nEmphasizing the Viash principles, `craftbox` components aim for **reusability**, **reproducibility**, and adherence to **best practices**. Key features generally include:\n\n* **Standalone & Nextflow Ready:** Components are built to run directly via the command line or be smoothly integrated into Nextflow workflows.\n* **Custom Implementations:** Contains scripts and tools developed for particular tasks that may not be found in broader collections.\n* **High Quality Standards (promoted by Viash):**\n * Clear documentation for components and their parameters.\n * Full exposure of underlying script/tool arguments for fine-grained control.\n * Containerized (Docker) to ensure dependency management and a consistent, reproducible runtime environment.\n * Unit tested where applicable to ensure components function as expected.\n",
"viash_version" : "0.9.4",
"viash_version" : "0.9.7",
"source" : "src",
"target" : "target",
"config_mods" : [

View File

@@ -181,8 +181,8 @@ build_info:
engine: "docker|native"
output: "target/nextflow/untar"
executable: "target/nextflow/untar/main.nf"
viash_version: "0.9.4"
git_commit: "32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550"
viash_version: "0.9.7"
git_commit: "b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1"
git_remote: "https://github.com/viash-hub/craftbox"
package_config:
name: "craftbox"
@@ -202,7 +202,7 @@ package_config:
\ to ensure dependency management and a consistent, reproducible runtime environment.\n\
\ * Unit tested where applicable to ensure components function as expected.\n"
info: null
viash_version: "0.9.4"
viash_version: "0.9.7"
source: "src"
target: "target"
config_mods:

View File

@@ -1,6 +1,6 @@
// untar main
//
// This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative
// This wrapper script is auto-generated by viash 0.9.7 and is thus a derivative
// work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data
// Intuitive.
//
@@ -1354,47 +1354,42 @@ def readCsv(file_path) {
def splitRegex = java.util.regex.Pattern.compile(''',(?=(?:[^"]*"[^"]*")*[^"]*$)''')
def removeQuote = java.util.regex.Pattern.compile('''"(.*)"''')
def br = java.nio.file.Files.newBufferedReader(inputFile)
java.nio.file.Files.newBufferedReader(inputFile).withCloseable { br ->
def row = 0
def header = null
def line
def row = -1
def header = null
while (br.ready() && header == null) {
def line = br.readLine()
row++
if (!line.startsWith("#")) {
header = splitRegex.split(line, -1).collect{field ->
m = removeQuote.matcher(field)
m.find() ? m.replaceFirst('$1') : field
}
}
}
assert header != null: "CSV file should contain a header"
while (br.ready()) {
def line = br.readLine()
row++
if (line == null) {
br.close()
break
}
if (!line.startsWith("#")) {
def predata = splitRegex.split(line, -1)
def data = predata.collect{field ->
if (field == "") {
return null
}
def m = removeQuote.matcher(field)
if (m.find()) {
return m.replaceFirst('$1')
} else {
return field
while (header == null && (line = br.readLine()) != null) {
if (!line.startsWith("#")) {
header = splitRegex.split(line, -1).collect { field ->
def m = removeQuote.matcher(field)
m.find() ? m.replaceFirst('$1') : field
}
}
assert header.size() == data.size(): "Row $row should contain the same number as fields as the header"
def dataMap = [header, data].transpose().collectEntries().findAll{it.value != null}
output.add(dataMap)
row++
}
assert header != null : "CSV file should contain a header"
while ((line = br.readLine()) != null) {
row++
if (!line.startsWith("#")) {
def predata = splitRegex.split(line, -1)
def data = predata.collect { field ->
if (field == "") {
return null
}
def m = removeQuote.matcher(field)
if (m.find()) {
return m.replaceFirst('$1')
} else {
return field
}
}
assert header.size() == data.size() : "Row $row should contain the same number as fields as the header"
def dataMap = [header, data].transpose().collectEntries().findAll { it.value != null }
output.add(dataMap)
}
}
}
@@ -1710,10 +1705,25 @@ process publishFilesProc {
]
.transpose()
.collectMany{infile, outfile ->
if (infile.toString() != outfile.toString()) {
def infileString = infile.toString()
def outfileString = outfile.toString()
if (infileString != outfileString) {
/* Trailing slashes are removed from both the source and destination arguments.
From source arguments, this is useful when a source argument may have a trailing slash
and specify a symbolic link to a directory. Without removing the slash, cp will dereference
the symbolic link.
See https://www.gnu.org/software/coreutils/manual/html_node/Trailing-slashes.html#Trailing-slashes-1
For the destination path addding a trailing slash is a problem when publishing directories:
it requires the destination directory to exist. This fails because we only create the parent
directories first.
*/
def regexTrailingSlashes = ~/\/+$/
def infileNoTrailingSlash = infileString - regexTrailingSlashes
def outfileNoTrailingSlash = outfileString - regexTrailingSlashes
[
"[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"",
"cp -r '${infile.toString()}' '${outfile.toString()}'"
"[ -d \"\$(dirname '${outfileNoTrailingSlash}')\" ] || mkdir -p \"\$(dirname '${outfileNoTrailingSlash}')\"",
"cp -a '${infileNoTrailingSlash}' '${outfileNoTrailingSlash}'"
]
} else {
// no need to copy if infile is the same as outfile
@@ -3272,8 +3282,8 @@ meta = [
"runner" : "nextflow",
"engine" : "docker|native",
"output" : "target/nextflow/untar",
"viash_version" : "0.9.4",
"git_commit" : "32cec141fe0eb61ce1b63f36a2e5c3ae2fbf9550",
"viash_version" : "0.9.7",
"git_commit" : "b74f55253eaa60d4dcd40a80b6965bc0bc9e1dd1",
"git_remote" : "https://github.com/viash-hub/craftbox"
},
"package_config" : {
@@ -3281,7 +3291,7 @@ meta = [
"version" : "main",
"summary" : "A collection of custom-tailored scripts and applied utilities built with Viash.\n",
"description" : "`craftbox` is a curated collection of custom scripts and utilities designed to tackle context-specific tasks.\n\nEmphasizing the Viash principles, `craftbox` components aim for **reusability**, **reproducibility**, and adherence to **best practices**. Key features generally include:\n\n* **Standalone & Nextflow Ready:** Components are built to run directly via the command line or be smoothly integrated into Nextflow workflows.\n* **Custom Implementations:** Contains scripts and tools developed for particular tasks that may not be found in broader collections.\n* **High Quality Standards (promoted by Viash):**\n * Clear documentation for components and their parameters.\n * Full exposure of underlying script/tool arguments for fine-grained control.\n * Containerized (Docker) to ensure dependency management and a consistent, reproducible runtime environment.\n * Unit tested where applicable to ensure components function as expected.\n",
"viash_version" : "0.9.4",
"viash_version" : "0.9.7",
"source" : "src",
"target" : "target",
"config_mods" : [