From b5f42c80555ac3ff7adbc31891a88ad4615688ee Mon Sep 17 00:00:00 2001 From: CI Date: Fri, 25 Apr 2025 12:27:03 +0000 Subject: [PATCH] Build branch v0.3 with version v0.3.9 (820318c) Build pipeline: viash-hub.demultiplex.v0.3-5lwbd Source commit: https://github.com/viash-hub/demultiplex/commit/820318c378d8119e2aa98768282e43c2aa017ba7 Source message: Bump version to v0.3.9 --- CHANGELOG.md | 12 + README.md | 6 +- _viash.yaml | 6 +- .../interop_summary_to_csv/.config.vsh.yaml | 23 +- .../interop_summary_to_csv | 76 ++- target/executable/io/publish/.config.vsh.yaml | 21 +- target/executable/io/publish/publish | 120 +++-- target/executable/io/untar/.config.vsh.yaml | 21 +- target/executable/io/untar/untar | 92 ++-- .../dataflow/combine_samples/.config.vsh.yaml | 19 +- .../nextflow/dataflow/combine_samples/main.nf | 456 ++++++++++++---- .../dataflow/combine_samples/nextflow.config | 2 +- .../combine_samples/nextflow_schema.json | 18 +- .../.config.vsh.yaml | 19 +- .../gather_fastqs_and_validate/main.nf | 456 ++++++++++++---- .../nextflow.config | 2 +- .../nextflow_schema.json | 12 +- target/nextflow/demultiplex/.config.vsh.yaml | 19 +- target/nextflow/demultiplex/main.nf | 456 ++++++++++++---- target/nextflow/demultiplex/nextflow.config | 2 +- .../nextflow/demultiplex/nextflow_schema.json | 24 +- .../interop_summary_to_csv/.config.vsh.yaml | 23 +- .../io/interop_summary_to_csv/main.nf | 485 +++++++++++++----- .../io/interop_summary_to_csv/nextflow.config | 2 +- .../nextflow_schema.json | 12 +- target/nextflow/io/publish/.config.vsh.yaml | 21 +- target/nextflow/io/publish/main.nf | 483 ++++++++++++----- target/nextflow/io/publish/nextflow.config | 2 +- .../nextflow/io/publish/nextflow_schema.json | 24 +- target/nextflow/io/untar/.config.vsh.yaml | 21 +- target/nextflow/io/untar/main.nf | 483 ++++++++++++----- target/nextflow/io/untar/nextflow.config | 2 +- target/nextflow/io/untar/nextflow_schema.json | 6 +- target/nextflow/runner/.config.vsh.yaml | 19 +- target/nextflow/runner/main.nf | 456 ++++++++++++---- target/nextflow/runner/nextflow.config | 2 +- target/nextflow/runner/nextflow_schema.json | 18 +- 37 files changed, 2860 insertions(+), 1061 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46e5be4..21e1bda 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +# demultiplex v0.3.9 + +## Bug fixes + +* Fix defaults for output arguments in nextflow schema's. + +* Fix an issue where an integer being passed to a argument with `type: double` resulted in an error (PR #44). + +## Minor changes + +* Bump viash to 0.9.4, which adds support for nextflow versions starting major version 25.01 (PR #43 and #44). + # demultiplex v0.3.8 ## Bug fixes diff --git a/README.md b/README.md index ece4e5e..f20a56e 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ License](https://img.shields.io/github/license/viash-hub/demultiplex.svg)](https [![GitHub Issues](https://img.shields.io/github/issues/viash-hub/demultiplex.svg)](https://github.com/viash-hub/demultiplex/issues) [![Viash -version](https://img.shields.io/badge/Viash-v0.9.1-blue)](https://viash.io) +version](https://img.shields.io/badge/Viash-v0.9.4-blue)](https://viash.io) ## Workflow Overview The workflow executes the following steps: @@ -53,7 +53,7 @@ You can check if everything is working by getting the `--help` for a workflow: ```bash nextflow run \ vsh/demultiplex \ --r v0.3.4 \ +-r v0.3.9 \ --help ``` @@ -84,7 +84,7 @@ When starting nextflow using the CLI, you can use `-c` to provide the file to ne ```bash nextflow run vsh/demultiplex \ --r v0.3.4 \ +-r v0.3.9 \ -main-script target/nextflow/runner/main.nf \ --input "gs://viash-hub-test-data/demultiplex/v3/demultiplex_htrnaseq_meta/SingleCell-RNA_P3_2" \ --demultiplexer bclconvert \ diff --git a/_viash.yaml b/_viash.yaml index 8c0f9c7..bcac3ee 100644 --- a/_viash.yaml +++ b/_viash.yaml @@ -1,5 +1,5 @@ name: demultiplex -version: v0.3.8 +version: v0.3.9 description: | Demultiplexing pipeline license: MIT @@ -12,10 +12,10 @@ info: - path: gs://viash-hub-test-data/demultiplex/v2/ dest: testData -viash_version: 0.9.0 +viash_version: 0.9.4 config_mods: | - .requirements.commands := ['ps'] + .requirements.commands += ['ps'] .runners[.type == 'nextflow'].directives.tag := '$id' .resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'} .runners[.type == 'nextflow'].config.script := 'includeConfig("nextflow_labels.config")' diff --git a/target/executable/io/interop_summary_to_csv/.config.vsh.yaml b/target/executable/io/interop_summary_to_csv/.config.vsh.yaml index 246c900..f22dc23 100644 --- a/target/executable/io/interop_summary_to_csv/.config.vsh.yaml +++ b/target/executable/io/interop_summary_to_csv/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "interop_summary_to_csv" namespace: "io" -version: "v0.3.8" +version: "v0.3.9" argument_groups: - name: "Input arguments" arguments: @@ -43,8 +43,13 @@ resources: dest: "nextflow_labels.config" info: null status: "enabled" +scope: + image: "public" + target: "public" requirements: commands: + - "summary" + - "index-summary" - "ps" license: "MIT" links: @@ -121,7 +126,7 @@ engines: id: "docker" image: "debian:stable-slim" target_registry: "images.viash-hub.com" - target_tag: "v0.3.8" + target_tag: "v0.3.9" namespace_separator: "/" setup: - type: "apt" @@ -145,29 +150,29 @@ build_info: engine: "docker|native" output: "target/executable/io/interop_summary_to_csv" executable: "target/executable/io/interop_summary_to_csv/interop_summary_to_csv" - viash_version: "0.9.0" - git_commit: "dafe2d829079bd5a9f5eb4e7cf63e92edbf1a742" + viash_version: "0.9.4" + git_commit: "820318c378d8119e2aa98768282e43c2aa017ba7" git_remote: "https://github.com/viash-hub/demultiplex" - git_tag: "v0.3.5-9-gdafe2d8" + git_tag: "v0.3.8-5-g820318c" package_config: name: "demultiplex" - version: "v0.3.8" + version: "v0.3.9" description: "Demultiplexing pipeline\n" info: test_resources: - path: "gs://viash-hub-test-data/demultiplex/v2/" dest: "testData" - viash_version: "0.9.0" + viash_version: "0.9.4" source: "src" target: "target" config_mods: - - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\ + - ".requirements.commands += ['ps']\n.runners[.type == 'nextflow'].directives.tag\ \ := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n\ .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ )'\n" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.3.8'" + - ".engines[.type == 'docker'].target_tag := 'v0.3.9'" keywords: - "bioinformatics" - "sequence" diff --git a/target/executable/io/interop_summary_to_csv/interop_summary_to_csv b/target/executable/io/interop_summary_to_csv/interop_summary_to_csv index 3098934..ee6717b 100755 --- a/target/executable/io/interop_summary_to_csv/interop_summary_to_csv +++ b/target/executable/io/interop_summary_to_csv/interop_summary_to_csv @@ -1,8 +1,8 @@ #!/usr/bin/env bash -# interop_summary_to_csv v0.3.8 +# interop_summary_to_csv v0.3.9 # -# This wrapper script is auto-generated by viash 0.9.0 and is thus a derivative +# This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative # work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data # Intuitive. # @@ -169,22 +169,6 @@ VIASH_META_CONFIG="$VIASH_META_RESOURCES_DIR/.config.vsh.yaml" VIASH_META_TEMP_DIR="$VIASH_TEMP" -# ViashHelp: Display helpful explanation about this executable -function ViashHelp { - echo "interop_summary_to_csv v0.3.8" - echo "" - echo "Input arguments:" - echo " --input" - echo " type: file, required parameter, file must exist" - echo " Sequencing run folder (*not* InterOp folder)." - echo "" - echo "Output arguments:" - echo " --output_run_summary" - echo " type: file, required parameter, output, file must exist" - echo "" - echo " --output_index_summary" - echo " type: file, required parameter, output, file must exist" -} # initialise variables VIASH_MODE='run' @@ -470,10 +454,10 @@ tar -C /tmp/ --no-same-owner --no-same-permissions -xvf /tmp/interop.tar.gz && \ mv /tmp/interop-1.3.1-Linux-GNU/bin/index-summary /tmp/interop-1.3.1-Linux-GNU/bin/summary /usr/local/bin/ LABEL org.opencontainers.image.description="Companion container for running component io interop_summary_to_csv" -LABEL org.opencontainers.image.created="2025-03-27T16:00:55Z" +LABEL org.opencontainers.image.created="2025-04-25T12:13:39Z" LABEL org.opencontainers.image.source="https://github.com/viash-hub/demultiplex" -LABEL org.opencontainers.image.revision="dafe2d829079bd5a9f5eb4e7cf63e92edbf1a742" -LABEL org.opencontainers.image.version="v0.3.8" +LABEL org.opencontainers.image.revision="820318c378d8119e2aa98768282e43c2aa017ba7" +LABEL org.opencontainers.image.version="v0.3.9" VIASHDOCKER fi @@ -587,6 +571,48 @@ fi # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) + +# ViashHelp: Display helpful explanation about this executable +function ViashHelp { + echo "interop_summary_to_csv v0.3.9" + echo "" + echo "Input arguments:" + echo " --input" + echo " type: file, required parameter, file must exist" + echo " Sequencing run folder (*not* InterOp folder)." + echo "" + echo "Output arguments:" + echo " --output_run_summary" + echo " type: file, required parameter, output, file must exist" + echo "" + echo " --output_index_summary" + echo " type: file, required parameter, output, file must exist" + echo "" + echo "Viash built in Computational Requirements:" + echo " ---cpus=INT" + echo " Number of CPUs to use" + echo " ---memory=STRING" + echo " Amount of memory to use. Examples: 4GB, 3MiB." + echo "" + echo "Viash built in Docker:" + echo " ---setup=STRATEGY" + echo " Setup the docker container. Options are: alwaysbuild, alwayscachedbuild, ifneedbebuild, ifneedbecachedbuild, alwayspull, alwayspullelsebuild, alwayspullelsecachedbuild, ifneedbepull, ifneedbepullelsebuild, ifneedbepullelsecachedbuild, push, pushifnotpresent, donothing." + echo " Default: ifneedbepullelsecachedbuild" + echo " ---dockerfile" + echo " Print the dockerfile to stdout." + echo " ---docker_run_args=ARG" + echo " Provide runtime arguments to Docker. See the documentation on \`docker run\` for more information." + echo " ---docker_image_id" + echo " Print the docker image id to stdout." + echo " ---debug" + echo " Enter the docker container for debugging purposes." + echo "" + echo "Viash built in Engines:" + echo " ---engine=ENGINE_ID" + echo " Specify the engine to use. Options are: docker, native." + echo " Default: docker" +} + # initialise array VIASH_POSITIONAL_ARGS='' @@ -609,7 +635,7 @@ while [[ $# -gt 0 ]]; do shift 1 ;; --version) - echo "interop_summary_to_csv v0.3.8" + echo "interop_summary_to_csv v0.3.9" exit ;; --input) @@ -733,7 +759,7 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then # determine docker image id if [[ "$VIASH_ENGINE_ID" == 'docker' ]]; then - VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/demultiplex/io/interop_summary_to_csv:v0.3.8' + VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/demultiplex/io/interop_summary_to_csv:v0.3.9' fi # print dockerfile @@ -755,13 +781,13 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then # build docker image elif [ "$VIASH_MODE" == "setup" ]; then ViashDockerSetup "$VIASH_DOCKER_IMAGE_ID" "$VIASH_SETUP_STRATEGY" - ViashDockerCheckCommands "$VIASH_DOCKER_IMAGE_ID" 'ps' 'bash' + ViashDockerCheckCommands "$VIASH_DOCKER_IMAGE_ID" 'summary' 'index-summary' 'ps' 'bash' exit 0 fi # check if docker image exists ViashDockerSetup "$VIASH_DOCKER_IMAGE_ID" ifneedbepullelsecachedbuild - ViashDockerCheckCommands "$VIASH_DOCKER_IMAGE_ID" 'ps' 'bash' + ViashDockerCheckCommands "$VIASH_DOCKER_IMAGE_ID" 'summary' 'index-summary' 'ps' 'bash' fi # setting computational defaults diff --git a/target/executable/io/publish/.config.vsh.yaml b/target/executable/io/publish/.config.vsh.yaml index cf36e40..c462aec 100644 --- a/target/executable/io/publish/.config.vsh.yaml +++ b/target/executable/io/publish/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "publish" namespace: "io" -version: "v0.3.8" +version: "v0.3.9" argument_groups: - name: "Input arguments" arguments: @@ -100,6 +100,9 @@ resources: description: "Publish the processed results of the run" info: null status: "enabled" +scope: + image: "public" + target: "public" requirements: commands: - "ps" @@ -178,7 +181,7 @@ engines: id: "docker" image: "debian:stable-slim" target_registry: "images.viash-hub.com" - target_tag: "v0.3.8" + target_tag: "v0.3.9" namespace_separator: "/" setup: - type: "apt" @@ -195,29 +198,29 @@ build_info: engine: "docker|native" output: "target/executable/io/publish" executable: "target/executable/io/publish/publish" - viash_version: "0.9.0" - git_commit: "dafe2d829079bd5a9f5eb4e7cf63e92edbf1a742" + viash_version: "0.9.4" + git_commit: "820318c378d8119e2aa98768282e43c2aa017ba7" git_remote: "https://github.com/viash-hub/demultiplex" - git_tag: "v0.3.5-9-gdafe2d8" + git_tag: "v0.3.8-5-g820318c" package_config: name: "demultiplex" - version: "v0.3.8" + version: "v0.3.9" description: "Demultiplexing pipeline\n" info: test_resources: - path: "gs://viash-hub-test-data/demultiplex/v2/" dest: "testData" - viash_version: "0.9.0" + viash_version: "0.9.4" source: "src" target: "target" config_mods: - - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\ + - ".requirements.commands += ['ps']\n.runners[.type == 'nextflow'].directives.tag\ \ := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n\ .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ )'\n" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.3.8'" + - ".engines[.type == 'docker'].target_tag := 'v0.3.9'" keywords: - "bioinformatics" - "sequence" diff --git a/target/executable/io/publish/publish b/target/executable/io/publish/publish index 63d36d1..81f0a9d 100755 --- a/target/executable/io/publish/publish +++ b/target/executable/io/publish/publish @@ -1,8 +1,8 @@ #!/usr/bin/env bash -# publish v0.3.8 +# publish v0.3.9 # -# This wrapper script is auto-generated by viash 0.9.0 and is thus a derivative +# This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative # work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data # Intuitive. # @@ -169,46 +169,6 @@ VIASH_META_CONFIG="$VIASH_META_RESOURCES_DIR/.config.vsh.yaml" VIASH_META_TEMP_DIR="$VIASH_TEMP" -# ViashHelp: Display helpful explanation about this executable -function ViashHelp { - echo "publish v0.3.8" - echo "" - echo "Publish the processed results of the run" - echo "" - echo "Input arguments:" - echo " --input" - echo " type: file, required parameter, file must exist" - echo " Directory to write fastq data to" - echo "" - echo " --input_falco" - echo " type: file, required parameter, multiple values allowed, file must exist" - echo " Directory to write falco output to" - echo "" - echo " --input_multiqc" - echo " type: file, required parameter, file must exist" - echo " Location where to write the MultiQC report to." - echo "" - echo " --input_run_information" - echo " type: file, required parameter, file must exist" - echo " Location where to write the run information to." - echo "" - echo "Output arguments:" - echo " --output" - echo " type: file, output, file must exist" - echo " default: fastq" - echo "" - echo " --output_falco" - echo " type: file, output, file must exist" - echo " default: qc/fastqc" - echo "" - echo " --output_multiqc" - echo " type: file, output, file must exist" - echo " default: qc/multiqc_report.html" - echo "" - echo " --output_run_information" - echo " type: file, output, file must exist" - echo " default: run_information.csv" -} # initialise variables VIASH_MODE='run' @@ -490,10 +450,10 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* LABEL org.opencontainers.image.description="Companion container for running component io publish" -LABEL org.opencontainers.image.created="2025-03-27T16:00:54Z" +LABEL org.opencontainers.image.created="2025-04-25T12:13:38Z" LABEL org.opencontainers.image.source="https://github.com/viash-hub/demultiplex" -LABEL org.opencontainers.image.revision="dafe2d829079bd5a9f5eb4e7cf63e92edbf1a742" -LABEL org.opencontainers.image.version="v0.3.8" +LABEL org.opencontainers.image.revision="820318c378d8119e2aa98768282e43c2aa017ba7" +LABEL org.opencontainers.image.version="v0.3.9" VIASHDOCKER fi @@ -607,6 +567,72 @@ fi # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) + +# ViashHelp: Display helpful explanation about this executable +function ViashHelp { + echo "publish v0.3.9" + echo "" + echo "Publish the processed results of the run" + echo "" + echo "Input arguments:" + echo " --input" + echo " type: file, required parameter, file must exist" + echo " Directory to write fastq data to" + echo "" + echo " --input_falco" + echo " type: file, required parameter, multiple values allowed, file must exist" + echo " Directory to write falco output to" + echo "" + echo " --input_multiqc" + echo " type: file, required parameter, file must exist" + echo " Location where to write the MultiQC report to." + echo "" + echo " --input_run_information" + echo " type: file, required parameter, file must exist" + echo " Location where to write the run information to." + echo "" + echo "Output arguments:" + echo " --output" + echo " type: file, output, file must exist" + echo " default: fastq" + echo "" + echo " --output_falco" + echo " type: file, output, file must exist" + echo " default: qc/fastqc" + echo "" + echo " --output_multiqc" + echo " type: file, output, file must exist" + echo " default: qc/multiqc_report.html" + echo "" + echo " --output_run_information" + echo " type: file, output, file must exist" + echo " default: run_information.csv" + echo "" + echo "Viash built in Computational Requirements:" + echo " ---cpus=INT" + echo " Number of CPUs to use" + echo " ---memory=STRING" + echo " Amount of memory to use. Examples: 4GB, 3MiB." + echo "" + echo "Viash built in Docker:" + echo " ---setup=STRATEGY" + echo " Setup the docker container. Options are: alwaysbuild, alwayscachedbuild, ifneedbebuild, ifneedbecachedbuild, alwayspull, alwayspullelsebuild, alwayspullelsecachedbuild, ifneedbepull, ifneedbepullelsebuild, ifneedbepullelsecachedbuild, push, pushifnotpresent, donothing." + echo " Default: ifneedbepullelsecachedbuild" + echo " ---dockerfile" + echo " Print the dockerfile to stdout." + echo " ---docker_run_args=ARG" + echo " Provide runtime arguments to Docker. See the documentation on \`docker run\` for more information." + echo " ---docker_image_id" + echo " Print the docker image id to stdout." + echo " ---debug" + echo " Enter the docker container for debugging purposes." + echo "" + echo "Viash built in Engines:" + echo " ---engine=ENGINE_ID" + echo " Specify the engine to use. Options are: docker, native." + echo " Default: docker" +} + # initialise array VIASH_POSITIONAL_ARGS='' @@ -629,7 +655,7 @@ while [[ $# -gt 0 ]]; do shift 1 ;; --version) - echo "publish v0.3.8" + echo "publish v0.3.9" exit ;; --input) @@ -814,7 +840,7 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then # determine docker image id if [[ "$VIASH_ENGINE_ID" == 'docker' ]]; then - VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/demultiplex/io/publish:v0.3.8' + VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/demultiplex/io/publish:v0.3.9' fi # print dockerfile diff --git a/target/executable/io/untar/.config.vsh.yaml b/target/executable/io/untar/.config.vsh.yaml index d6307ef..be26292 100644 --- a/target/executable/io/untar/.config.vsh.yaml +++ b/target/executable/io/untar/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "untar" namespace: "io" -version: "v0.3.8" +version: "v0.3.9" argument_groups: - name: "Input arguments" arguments: @@ -57,6 +57,9 @@ test_resources: is_executable: true info: null status: "enabled" +scope: + image: "public" + target: "public" requirements: commands: - "ps" @@ -135,7 +138,7 @@ engines: id: "docker" image: "debian:stable-slim" target_registry: "images.viash-hub.com" - target_tag: "v0.3.8" + target_tag: "v0.3.9" namespace_separator: "/" setup: - type: "apt" @@ -152,29 +155,29 @@ build_info: engine: "docker|native" output: "target/executable/io/untar" executable: "target/executable/io/untar/untar" - viash_version: "0.9.0" - git_commit: "dafe2d829079bd5a9f5eb4e7cf63e92edbf1a742" + viash_version: "0.9.4" + git_commit: "820318c378d8119e2aa98768282e43c2aa017ba7" git_remote: "https://github.com/viash-hub/demultiplex" - git_tag: "v0.3.5-9-gdafe2d8" + git_tag: "v0.3.8-5-g820318c" package_config: name: "demultiplex" - version: "v0.3.8" + version: "v0.3.9" description: "Demultiplexing pipeline\n" info: test_resources: - path: "gs://viash-hub-test-data/demultiplex/v2/" dest: "testData" - viash_version: "0.9.0" + viash_version: "0.9.4" source: "src" target: "target" config_mods: - - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\ + - ".requirements.commands += ['ps']\n.runners[.type == 'nextflow'].directives.tag\ \ := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n\ .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ )'\n" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.3.8'" + - ".engines[.type == 'docker'].target_tag := 'v0.3.9'" keywords: - "bioinformatics" - "sequence" diff --git a/target/executable/io/untar/untar b/target/executable/io/untar/untar index af69f50..61e7b5d 100755 --- a/target/executable/io/untar/untar +++ b/target/executable/io/untar/untar @@ -1,8 +1,8 @@ #!/usr/bin/env bash -# untar v0.3.8 +# untar v0.3.9 # -# This wrapper script is auto-generated by viash 0.9.0 and is thus a derivative +# This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative # work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data # Intuitive. # @@ -169,32 +169,6 @@ VIASH_META_CONFIG="$VIASH_META_RESOURCES_DIR/.config.vsh.yaml" VIASH_META_TEMP_DIR="$VIASH_TEMP" -# ViashHelp: Display helpful explanation about this executable -function ViashHelp { - echo "untar v0.3.8" - echo "" - echo "Unpack a .tar file. When the contents of the .tar file is just a single" - echo "directory," - echo "put the contents of the directory into the output folder instead of that" - echo "directory." - echo "" - echo "Input arguments:" - echo " --input" - echo " type: file, required parameter, file must exist" - echo " Tarball file to be unpacked." - echo "" - echo "Output arguments:" - echo " --output" - echo " type: file, required parameter, output, file must exist" - echo " Directory to write the contents of the .tar file to." - echo "" - echo "Other arguments:" - echo " -e, --exclude" - echo " type: string" - echo " example: docs/figures" - echo " Prevents any file or member whose name matches the shell wildcard" - echo " (pattern) from being extracted." -} # initialise variables VIASH_MODE='run' @@ -476,10 +450,10 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* LABEL org.opencontainers.image.description="Companion container for running component io untar" -LABEL org.opencontainers.image.created="2025-03-27T16:00:54Z" +LABEL org.opencontainers.image.created="2025-04-25T12:13:38Z" LABEL org.opencontainers.image.source="https://github.com/viash-hub/demultiplex" -LABEL org.opencontainers.image.revision="dafe2d829079bd5a9f5eb4e7cf63e92edbf1a742" -LABEL org.opencontainers.image.version="v0.3.8" +LABEL org.opencontainers.image.revision="820318c378d8119e2aa98768282e43c2aa017ba7" +LABEL org.opencontainers.image.version="v0.3.9" VIASHDOCKER fi @@ -593,6 +567,58 @@ fi # initialise docker variables VIASH_DOCKER_RUN_ARGS=(-i --rm) + +# ViashHelp: Display helpful explanation about this executable +function ViashHelp { + echo "untar v0.3.9" + echo "" + echo "Unpack a .tar file. When the contents of the .tar file is just a single" + echo "directory," + echo "put the contents of the directory into the output folder instead of that" + echo "directory." + echo "" + echo "Input arguments:" + echo " --input" + echo " type: file, required parameter, file must exist" + echo " Tarball file to be unpacked." + echo "" + echo "Output arguments:" + echo " --output" + echo " type: file, required parameter, output, file must exist" + echo " Directory to write the contents of the .tar file to." + echo "" + echo "Other arguments:" + echo " -e, --exclude" + echo " type: string" + echo " example: docs/figures" + echo " Prevents any file or member whose name matches the shell wildcard" + echo " (pattern) from being extracted." + echo "" + echo "Viash built in Computational Requirements:" + echo " ---cpus=INT" + echo " Number of CPUs to use" + echo " ---memory=STRING" + echo " Amount of memory to use. Examples: 4GB, 3MiB." + echo "" + echo "Viash built in Docker:" + echo " ---setup=STRATEGY" + echo " Setup the docker container. Options are: alwaysbuild, alwayscachedbuild, ifneedbebuild, ifneedbecachedbuild, alwayspull, alwayspullelsebuild, alwayspullelsecachedbuild, ifneedbepull, ifneedbepullelsebuild, ifneedbepullelsecachedbuild, push, pushifnotpresent, donothing." + echo " Default: ifneedbepullelsecachedbuild" + echo " ---dockerfile" + echo " Print the dockerfile to stdout." + echo " ---docker_run_args=ARG" + echo " Provide runtime arguments to Docker. See the documentation on \`docker run\` for more information." + echo " ---docker_image_id" + echo " Print the docker image id to stdout." + echo " ---debug" + echo " Enter the docker container for debugging purposes." + echo "" + echo "Viash built in Engines:" + echo " ---engine=ENGINE_ID" + echo " Specify the engine to use. Options are: docker, native." + echo " Default: docker" +} + # initialise array VIASH_POSITIONAL_ARGS='' @@ -615,7 +641,7 @@ while [[ $# -gt 0 ]]; do shift 1 ;; --version) - echo "untar v0.3.8" + echo "untar v0.3.9" exit ;; --input) @@ -745,7 +771,7 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then # determine docker image id if [[ "$VIASH_ENGINE_ID" == 'docker' ]]; then - VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/demultiplex/io/untar:v0.3.8' + VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/demultiplex/io/untar:v0.3.9' fi # print dockerfile diff --git a/target/nextflow/dataflow/combine_samples/.config.vsh.yaml b/target/nextflow/dataflow/combine_samples/.config.vsh.yaml index 17de5b8..7b77dd5 100644 --- a/target/nextflow/dataflow/combine_samples/.config.vsh.yaml +++ b/target/nextflow/dataflow/combine_samples/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "combine_samples" namespace: "dataflow" -version: "v0.3.8" +version: "v0.3.9" argument_groups: - name: "Input arguments" arguments: @@ -80,6 +80,9 @@ description: "Combine fastq files from across samples into one event with a list \ fastq files per orientation." info: null status: "enabled" +scope: + image: "public" + target: "public" requirements: commands: - "ps" @@ -161,29 +164,29 @@ build_info: engine: "native|native" output: "target/nextflow/dataflow/combine_samples" executable: "target/nextflow/dataflow/combine_samples/main.nf" - viash_version: "0.9.0" - git_commit: "dafe2d829079bd5a9f5eb4e7cf63e92edbf1a742" + viash_version: "0.9.4" + git_commit: "820318c378d8119e2aa98768282e43c2aa017ba7" git_remote: "https://github.com/viash-hub/demultiplex" - git_tag: "v0.3.5-9-gdafe2d8" + git_tag: "v0.3.8-5-g820318c" package_config: name: "demultiplex" - version: "v0.3.8" + version: "v0.3.9" description: "Demultiplexing pipeline\n" info: test_resources: - path: "gs://viash-hub-test-data/demultiplex/v2/" dest: "testData" - viash_version: "0.9.0" + viash_version: "0.9.4" source: "src" target: "target" config_mods: - - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\ + - ".requirements.commands += ['ps']\n.runners[.type == 'nextflow'].directives.tag\ \ := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n\ .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ )'\n" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.3.8'" + - ".engines[.type == 'docker'].target_tag := 'v0.3.9'" keywords: - "bioinformatics" - "sequence" diff --git a/target/nextflow/dataflow/combine_samples/main.nf b/target/nextflow/dataflow/combine_samples/main.nf index 47404af..b0702da 100644 --- a/target/nextflow/dataflow/combine_samples/main.nf +++ b/target/nextflow/dataflow/combine_samples/main.nf @@ -1,6 +1,6 @@ -// combine_samples v0.3.8 +// combine_samples v0.3.9 // -// This wrapper script is auto-generated by viash 0.9.0 and is thus a derivative +// This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data // Intuitive. // @@ -82,64 +82,56 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi foundClass = "List[${e.foundClass}]" } } else if (par.type == "string") { - // cast to string if need be + // cast to string if need be. only cast if the value is a GString if (value instanceof GString) { - value = value.toString() + value = value as String } expectedClass = value instanceof String ? null : "String" } else if (par.type == "integer") { // cast to integer if need be - if (value instanceof String) { + if (value !instanceof Integer) { try { - value = value.toInteger() + value = value as Integer } catch (NumberFormatException e) { - // do nothing + expectedClass = "Integer" } } - if (value instanceof java.math.BigInteger) { - value = value.intValue() - } - expectedClass = value instanceof Integer ? null : "Integer" } else if (par.type == "long") { // cast to long if need be - if (value instanceof String) { + if (value !instanceof Long) { try { - value = value.toLong() + value = value as Long } catch (NumberFormatException e) { - // do nothing + expectedClass = "Long" } } - if (value instanceof Integer) { - value = value.toLong() - } - expectedClass = value instanceof Long ? null : "Long" } else if (par.type == "double") { // cast to double if need be - if (value instanceof String) { + if (value !instanceof Double) { try { - value = value.toDouble() + value = value as Double } catch (NumberFormatException e) { - // do nothing + expectedClass = "Double" } } - if (value instanceof java.math.BigDecimal) { - value = value.doubleValue() + } else if (par.type == "float") { + // cast to float if need be + if (value !instanceof Float) { + try { + value = value as Float + } catch (NumberFormatException e) { + expectedClass = "Float" + } } - if (value instanceof Float) { - value = value.toDouble() - } - expectedClass = value instanceof Double ? null : "Double" } else if (par.type == "boolean" | par.type == "boolean_true" | par.type == "boolean_false") { // cast to boolean if need be - if (value instanceof String) { - def valueLower = value.toLowerCase() - if (valueLower == "true") { - value = true - } else if (valueLower == "false") { - value = false + if (value !instanceof Boolean) { + try { + value = value as Boolean + } catch (Exception e) { + expectedClass = "Boolean" } } - expectedClass = value instanceof Boolean ? null : "Boolean" } else if (par.type == "file" && (par.direction == "input" || stage == "output")) { // cast to path if need be if (value instanceof String) { @@ -151,10 +143,13 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi expectedClass = value instanceof Path ? null : "Path" } else if (par.type == "file" && stage == "input" && par.direction == "output") { // cast to string if need be - if (value instanceof GString) { - value = value.toString() + if (value !instanceof String) { + try { + value = value as String + } catch (Exception e) { + expectedClass = "String" + } } - expectedClass = value instanceof String ? null : "String" } else { // didn't find a match for par.type expectedClass = par.type @@ -173,7 +168,7 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi Map _processInputValues(Map inputs, Map config, String id, String key) { if (!workflow.stubRun) { config.allArguments.each { arg -> - if (arg.required) { + if (arg.required && arg.direction == "input") { assert inputs.containsKey(arg.plainName) && inputs.get(arg.plainName) != null : "Error in module '${key}' id '${id}': required input argument '${arg.plainName}' is missing" } @@ -192,15 +187,8 @@ Map _processInputValues(Map inputs, Map config, String id, String key) { } // helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_processOutputValues.nf' -Map _processOutputValues(Map outputs, Map config, String id, String key) { +Map _checkValidOutputArgument(Map outputs, Map config, String id, String key) { if (!workflow.stubRun) { - config.allArguments.each { arg -> - if (arg.direction == "output" && arg.required) { - assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : - "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing" - } - } - outputs = outputs.collectEntries { name, value -> def par = config.allArguments.find { it.plainName == name && it.direction == "output" } assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid output argument" @@ -213,6 +201,16 @@ Map _processOutputValues(Map outputs, Map config, String id, String key) { return outputs } +void _checkAllRequiredOuputsPresent(Map outputs, Map config, String id, String key) { + if (!workflow.stubRun) { + config.allArguments.each { arg -> + if (arg.direction == "output" && arg.required) { + assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : + "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing" + } + } + } +} // helper file: 'src/main/resources/io/viash/runners/nextflow/channel/IDChecker.nf' class IDChecker { final def items = [] as Set @@ -1666,6 +1664,162 @@ def joinStates(Closure apply_) { } return joinStatesWf } +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishFiles.nf' +def publishFiles(Map args) { + def key_ = args.get("key") + + assert key_ != null : "publishFiles: key must be specified" + + workflow publishFilesWf { + take: input_ch + main: + input_ch + | map { tup -> + def id_ = tup[0] + def state_ = tup[1] + + // the input files and the target output filenames + def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose() + def inputFiles_ = inputoutputFilenames_[0] + def outputFilenames_ = inputoutputFilenames_[1] + + [id_, inputFiles_, outputFilenames_] + } + | publishFilesProc + emit: input_ch + } + return publishFilesWf +} + +process publishFilesProc { + // todo: check publishpath? + publishDir path: "${getPublishDir()}/", mode: "copy" + tag "$id" + input: + tuple val(id), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles) + output: + tuple val(id), path{outputFiles} + script: + def copyCommands = [ + inputFiles instanceof List ? inputFiles : [inputFiles], + outputFiles instanceof List ? outputFiles : [outputFiles] + ] + .transpose() + .collectMany{infile, outfile -> + if (infile.toString() != outfile.toString()) { + [ + "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"", + "cp -r '${infile.toString()}' '${outfile.toString()}'" + ] + } else { + // no need to copy if infile is the same as outfile + [] + } + } + """ + echo "Copying output files to destination folder" + ${copyCommands.join("\n ")} + """ +} + + +// this assumes that the state contains no other values other than those specified in the config +def publishFilesByConfig(Map args) { + def config = args.get("config") + assert config != null : "publishFilesByConfig: config must be specified" + + def key_ = args.get("key", config.name) + assert key_ != null : "publishFilesByConfig: key must be specified" + + workflow publishFilesSimpleWf { + take: input_ch + main: + input_ch + | map { tup -> + def id_ = tup[0] + def state_ = tup[1] // e.g. [output: new File("myoutput.h5ad"), k: 10] + def origState_ = tup[2] // e.g. [output: '$id.$key.foo.h5ad'] + + + // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where + // - key is a String + // - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path) + // - inputPath is a List[Path] + // - outputFilename is a List[String] + // - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml) + def processedState = + config.allArguments + .findAll { it.direction == "output" } + .collectMany { par -> + def plainName_ = par.plainName + // if the state does not contain the key, it's an + // optional argument for which the component did + // not generate any output OR multiple channels were emitted + // and the output was just not added to using the channel + // that is now being parsed + if (!state_.containsKey(plainName_)) { + return [] + } + def value = state_[plainName_] + // if the parameter is not a file, it should be stored + // in the state as-is, but is not something that needs + // to be copied from the source path to the dest path + if (par.type != "file") { + return [[inputPath: [], outputFilename: []]] + } + // if the orig state does not contain this filename, + // it's an optional argument for which the user specified + // that it should not be returned as a state + if (!origState_.containsKey(plainName_)) { + return [] + } + def filenameTemplate = origState_[plainName_] + // if the pararameter is multiple: true, fetch the template + if (par.multiple && filenameTemplate instanceof List) { + filenameTemplate = filenameTemplate[0] + } + // instantiate the template + def filename = filenameTemplate + .replaceAll('\\$id', id_) + .replaceAll('\\$\\{id\\}', id_) + .replaceAll('\\$key', key_) + .replaceAll('\\$\\{key\\}', key_) + if (par.multiple) { + // if the parameter is multiple: true, the filename + // should contain a wildcard '*' that is replaced with + // the index of the file + assert filename.contains("*") : "Module '${key_}' id '${id_}': Multiple output files specified, but no wildcard '*' in the filename: ${filename}" + def outputPerFile = value.withIndex().collect{ val, ix -> + def filename_ix = filename.replace("*", ix.toString()) + def inputPath = val instanceof File ? val.toPath() : val + [inputPath: inputPath, outputFilename: filename_ix] + } + def transposedOutputs = ["inputPath", "outputFilename"].collectEntries{ key -> + [key, outputPerFile.collect{dic -> dic[key]}] + } + return [[key: plainName_] + transposedOutputs] + } else { + def value_ = java.nio.file.Paths.get(filename) + def inputPath = value instanceof File ? value.toPath() : value + return [[inputPath: [inputPath], outputFilename: [filename]]] + } + } + + def inputPaths = processedState.collectMany{it.inputPath} + def outputFilenames = processedState.collectMany{it.outputFilename} + + + [id_, inputPaths, outputFilenames] + } + | publishFilesProc + emit: input_ch + } + return publishFilesSimpleWf +} + + + + // helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishStates.nf' def collectFiles(obj) { if (obj instanceof java.io.File || obj instanceof Path) { @@ -1723,8 +1877,6 @@ def publishStates(Map args) { // the input files and the target output filenames def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose() - def inputFiles_ = inputoutputFilenames_[0] - def outputFilenames_ = inputoutputFilenames_[1] def yamlFilename = yamlTemplate_ .replaceAll('\\$id', id_) @@ -1737,7 +1889,7 @@ def publishStates(Map args) { // convert state to yaml blob def yamlBlob_ = toRelativeTaggedYamlBlob([id: id_] + state_, java.nio.file.Paths.get(yamlFilename)) - [id_, yamlBlob_, yamlFilename, inputFiles_, outputFilenames_] + [id_, yamlBlob_, yamlFilename] } | publishStatesProc emit: input_ch @@ -1749,33 +1901,17 @@ process publishStatesProc { publishDir path: "${getPublishDir()}/", mode: "copy" tag "$id" input: - tuple val(id), val(yamlBlob), val(yamlFile), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles) + tuple val(id), val(yamlBlob), val(yamlFile) output: - tuple val(id), path{[yamlFile] + outputFiles} + tuple val(id), path{[yamlFile]} script: - def copyCommands = [ - inputFiles instanceof List ? inputFiles : [inputFiles], - outputFiles instanceof List ? outputFiles : [outputFiles] - ] - .transpose() - .collectMany{infile, outfile -> - if (infile.toString() != outfile.toString()) { - [ - "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"", - "cp -r '${infile.toString()}' '${outfile.toString()}'" - ] - } else { - // no need to copy if infile is the same as outfile - [] - } - } """ -mkdir -p "\$(dirname '${yamlFile}')" -echo "Storing state as yaml" -echo '${yamlBlob}' > '${yamlFile}' -echo "Copying output files to destination folder" -${copyCommands.join("\n ")} -""" + mkdir -p "\$(dirname '${yamlFile}')" + echo "Storing state as yaml" + cat > '${yamlFile}' << HERE +${yamlBlob} +HERE + """ } @@ -1806,13 +1942,10 @@ def publishStatesByConfig(Map args) { .replaceAll('\\$\\{key\\}', key_) def yamlDir = java.nio.file.Paths.get(yamlFilename).getParent() - // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where + // the processed state is a list of [key, value] tuples, where // - key is a String // - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path) - // - inputPath is a List[Path] - // - outputFilename is a List[String] // - (key, value) are the tuples that will be saved to the state.yaml file - // - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml) def processedState = config.allArguments .findAll { it.direction == "output" } @@ -1829,7 +1962,7 @@ def publishStatesByConfig(Map args) { // in the state as-is, but is not something that needs // to be copied from the source path to the dest path if (par.type != "file") { - return [[key: plainName_, value: value, inputPath: [], outputFilename: []]] + return [[key: plainName_, value: value]] } // if the orig state does not contain this filename, // it's an optional argument for which the user specified @@ -1860,13 +1993,9 @@ def publishStatesByConfig(Map args) { if (yamlDir != null) { value_ = yamlDir.relativize(value_) } - def inputPath = val instanceof File ? val.toPath() : val - [value: value_, inputPath: inputPath, outputFilename: filename_ix] + return value_ } - def transposedOutputs = ["value", "inputPath", "outputFilename"].collectEntries{ key -> - [key, outputPerFile.collect{dic -> dic[key]}] - } - return [[key: plainName_] + transposedOutputs] + return [["key": plainName_, "value": outputPerFile]] } else { def value_ = java.nio.file.Paths.get(filename) // if id contains a slash @@ -1874,18 +2003,17 @@ def publishStatesByConfig(Map args) { value_ = yamlDir.relativize(value_) } def inputPath = value instanceof File ? value.toPath() : value - return [[key: plainName_, value: value_, inputPath: [inputPath], outputFilename: [filename]]] + return [["key": plainName_, value: value_]] } } + def updatedState_ = processedState.collectEntries{[it.key, it.value]} - def inputPaths = processedState.collectMany{it.inputPath} - def outputFilenames = processedState.collectMany{it.outputFilename} // convert state to yaml blob def yamlBlob_ = toTaggedYamlBlob([id: id_] + updatedState_) - [id_, yamlBlob_, yamlFilename, inputPaths, outputFilenames] + [id_, yamlBlob_, yamlFilename] } | publishStatesProc emit: input_ch @@ -2559,7 +2687,8 @@ def _debug(workflowArgs, debugKey) { def workflowFactory(Map args, Map defaultWfArgs, Map meta) { def workflowArgs = processWorkflowArgs(args, defaultWfArgs, meta) def key_ = workflowArgs["key"] - + def multipleArgs = meta.config.allArguments.findAll{ it.multiple }.collect{it.plainName} + workflow workflowInstance { take: input_ @@ -2716,12 +2845,36 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { } // TODO: move some of the _meta.join_id wrangling to the safeJoin() function. - def chInitialOutput = chArgsWithDefaults + def chInitialOutputMulti = chArgsWithDefaults | _debug(workflowArgs, "processed") // run workflow | innerWorkflowFactory(workflowArgs) - // check output tuple - | map { id_, output_ -> + def chInitialOutputList = chInitialOutputMulti instanceof List ? chInitialOutputMulti : [chInitialOutputMulti] + assert chInitialOutputList.size() > 0: "should have emitted at least one output channel" + // Add a channel ID to the events, which designates the channel the event was emitted from as a running number + // This number is used to sort the events later when the events are gathered from across the channels. + def chInitialOutputListWithIndexedEvents = chInitialOutputList.withIndex().collect{channel, channelIndex -> + def newChannel = channel + | map {tuple -> + assert tuple instanceof List : + "Error in module '${key_}': element in output channel should be a tuple [id, data, ...otherargs...]\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Expected class: List. Found: tuple.getClass() is ${tuple.getClass()}" + + def newEvent = [channelIndex] + tuple + return newEvent + } + return newChannel + } + // Put the events into 1 channel, cover case where there is only one channel is emitted + def chInitialOutput = chInitialOutputList.size() > 1 ? \ + chInitialOutputListWithIndexedEvents[0].mix(*chInitialOutputListWithIndexedEvents.tail()) : \ + chInitialOutputListWithIndexedEvents[0] + def chInitialOutputProcessed = chInitialOutput + | map { tuple -> + def channelId = tuple[0] + def id_ = tuple[1] + def output_ = tuple[2] // see if output map contains metadata def meta_ = @@ -2734,19 +2887,94 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { output_ = output_.findAll{k, v -> k != "_meta"} // check value types - output_ = _processOutputValues(output_, meta.config, id_, key_) + output_ = _checkValidOutputArgument(output_, meta.config, id_, key_) - // simplify output if need be - if (workflowArgs.auto.simplifyOutput && output_.size() == 1) { - output_ = output_.values()[0] - } - - [join_id, id_, output_] + [join_id, channelId, id_, output_] } // | view{"chInitialOutput: ${it.take(3)}"} + // join the output [prev_id, channel_id, new_id, output] with the previous state [prev_id, state, ...] + def chPublishWithPreviousState = safeJoin(chInitialOutputProcessed, chRunFiltered, key_) + // input tuple format: [join_id, channel_id, id, output, prev_state, ...] + // output tuple format: [join_id, channel_id, id, new_state, ...] + | map{ tup -> + def new_state = workflowArgs.toState(tup.drop(2).take(3)) + tup.take(3) + [new_state] + tup.drop(5) + } + if (workflowArgs.auto.publish == "state") { + def chPublishFiles = chPublishWithPreviousState + // input tuple format: [join_id, channel_id, id, new_state, ...] + // output tuple format: [join_id, channel_id, id, new_state] + | map{ tup -> + tup.take(4) + } + + safeJoin(chPublishFiles, chArgsWithDefaults, key_) + // input tuple format: [join_id, channel_id, id, new_state, orig_state, ...] + // output tuple format: [id, new_state, orig_state] + | map { tup -> + tup.drop(2).take(3) + } + | publishFilesByConfig(key: key_, config: meta.config) + } + // Join the state from the events that were emitted from different channels + def chJoined = chInitialOutputProcessed + | map {tuple -> + def join_id = tuple[0] + def channel_id = tuple[1] + def id = tuple[2] + def other = tuple.drop(3) + // Below, groupTuple is used to join the events. To make sure resuming a workflow + // keeps working, the output state must be deterministic. This means the state needs to be + // sorted with groupTuple's has a 'sort' argument. This argument can be set to 'hash', + // but hashing the state when it is large can be problematic in terms of performance. + // Therefore, a custom comparator function is provided. We add the channel ID to the + // states so that we can use the channel ID to sort the items. + def stateWithChannelID = [[channel_id] * other.size(), other].transpose() + // A comparator that is provided to groupTuple's 'sort' argument is applied + // to all elements of the event tuple (that is not the 'id'). The comparator + // closure that is used below expects the input to be List. So the join_id and + // channel_id must also be wrapped in a list. + [[join_id], [channel_id], id] + stateWithChannelID + } + | groupTuple(by: 2, sort: {a, b -> a[0] <=> b[0]}, size: chInitialOutputList.size(), remainder: true) + | map {join_ids, _, id, statesWithChannelID -> + // Remove the channel IDs from the states + def states = statesWithChannelID.collect{it[1]} + def newJoinId = join_ids.flatten().unique{a, b -> a <=> b} + assert newJoinId.size() == 1: "Multiple events were emitted for '$id'." + def newJoinIdUnique = newJoinId[0] + + // Merge the states from the different channels + def newState = states.inject([:]){ old_state, state_to_add -> + return old_state + state_to_add.collectEntries{k, v -> + if (!multipleArgs.contains(k)) { + // if the key is not a multiple argument, we expect only one value + if (old_state.containsKey(k)) { + assert old_state[k] == v : "ID $id: multiple entries for argument $k were emitted." + } + [k, v] + } else { + // if the key is a multiple argument, append the different values into one list + def prevValue = old_state.getOrDefault(k, []) + def prevValueAsList = prevValue instanceof List ? prevValue : [prevValue] + [k, prevValueAsList + v] + } + } + } + + _checkAllRequiredOuputsPresent(newState, meta.config, id, key_) + + // simplify output if need be + if (workflowArgs.auto.simplifyOutput && newState.size() == 1) { + newState = newState.values()[0] + } + + return [newJoinIdUnique, id, newState] + } + // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) + def chNewState = safeJoin(chJoined, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -2755,23 +2983,21 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { } if (workflowArgs.auto.publish == "state") { - def chPublish = chNewState + def chPublishStates = chNewState // input tuple format: [join_id, id, new_state, ...] // output tuple format: [join_id, id, new_state] | map{ tup -> tup.take(3) } - safeJoin(chPublish, chArgsWithDefaults, key_) + safeJoin(chPublishStates, chArgsWithDefaults, key_) // input tuple format: [join_id, id, new_state, orig_state, ...] // output tuple format: [id, new_state, orig_state] | map { tup -> tup.drop(1).take(3) - } + } | publishStatesByConfig(key: key_, config: meta.config) } - - // remove join_id and meta chReturn = chNewState | map { tup -> // input tuple format: [join_id, id, new_state, ...] @@ -2806,7 +3032,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "combine_samples", "namespace" : "dataflow", - "version" : "v0.3.8", + "version" : "v0.3.9", "argument_groups" : [ { "name" : "Input arguments", @@ -2903,6 +3129,10 @@ meta = [ ], "description" : "Combine fastq files from across samples into one event with a list of fastq files per orientation.", "status" : "enabled", + "scope" : { + "image" : "public", + "target" : "public" + }, "requirements" : { "commands" : [ "ps" @@ -2999,14 +3229,14 @@ meta = [ "runner" : "nextflow", "engine" : "native|native", "output" : "target/nextflow/dataflow/combine_samples", - "viash_version" : "0.9.0", - "git_commit" : "dafe2d829079bd5a9f5eb4e7cf63e92edbf1a742", + "viash_version" : "0.9.4", + "git_commit" : "820318c378d8119e2aa98768282e43c2aa017ba7", "git_remote" : "https://github.com/viash-hub/demultiplex", - "git_tag" : "v0.3.5-9-gdafe2d8" + "git_tag" : "v0.3.8-5-g820318c" }, "package_config" : { "name" : "demultiplex", - "version" : "v0.3.8", + "version" : "v0.3.9", "description" : "Demultiplexing pipeline\n", "info" : { "test_resources" : [ @@ -3016,14 +3246,14 @@ meta = [ } ] }, - "viash_version" : "0.9.0", + "viash_version" : "0.9.4", "source" : "src", "target" : "target", "config_mods" : [ - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n", + ".requirements.commands += ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v0.3.8'" + ".engines[.type == 'docker'].target_tag := 'v0.3.9'" ], "keywords" : [ "bioinformatics", diff --git a/target/nextflow/dataflow/combine_samples/nextflow.config b/target/nextflow/dataflow/combine_samples/nextflow.config index 15037e4..a6a2963 100644 --- a/target/nextflow/dataflow/combine_samples/nextflow.config +++ b/target/nextflow/dataflow/combine_samples/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'dataflow/combine_samples' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v0.3.8' + version = 'v0.3.9' description = 'Combine fastq files from across samples into one event with a list of fastq files per orientation.' } diff --git a/target/nextflow/dataflow/combine_samples/nextflow_schema.json b/target/nextflow/dataflow/combine_samples/nextflow_schema.json index 26f41f2..290ce6e 100644 --- a/target/nextflow/dataflow/combine_samples/nextflow_schema.json +++ b/target/nextflow/dataflow/combine_samples/nextflow_schema.json @@ -67,10 +67,10 @@ "output_forward": { "type": "string", - "description": "Type: List of `file`, required, default: `$id.$key.output_forward_*.output_forward_*`, multiple_sep: `\";\"`. ", - "help_text": "Type: List of `file`, required, default: `$id.$key.output_forward_*.output_forward_*`, multiple_sep: `\";\"`. " + "description": "Type: List of `file`, required, default: `$id.$key.output_forward_*`, multiple_sep: `\";\"`. ", + "help_text": "Type: List of `file`, required, default: `$id.$key.output_forward_*`, multiple_sep: `\";\"`. " , - "default":"$id.$key.output_forward_*.output_forward_*" + "default":"$id.$key.output_forward_*" } @@ -78,10 +78,10 @@ "output_reverse": { "type": "string", - "description": "Type: List of `file`, default: `$id.$key.output_reverse_*.output_reverse_*`, multiple_sep: `\";\"`. ", - "help_text": "Type: List of `file`, default: `$id.$key.output_reverse_*.output_reverse_*`, multiple_sep: `\";\"`. " + "description": "Type: List of `file`, default: `$id.$key.output_reverse_*`, multiple_sep: `\";\"`. ", + "help_text": "Type: List of `file`, default: `$id.$key.output_reverse_*`, multiple_sep: `\";\"`. " , - "default":"$id.$key.output_reverse_*.output_reverse_*" + "default":"$id.$key.output_reverse_*" } @@ -89,10 +89,10 @@ "output_falco": { "type": "string", - "description": "Type: List of `file`, required, default: `$id.$key.output_falco_*.output_falco_*`, multiple_sep: `\";\"`. ", - "help_text": "Type: List of `file`, required, default: `$id.$key.output_falco_*.output_falco_*`, multiple_sep: `\";\"`. " + "description": "Type: List of `file`, required, default: `$id.$key.output_falco_*`, multiple_sep: `\";\"`. ", + "help_text": "Type: List of `file`, required, default: `$id.$key.output_falco_*`, multiple_sep: `\";\"`. " , - "default":"$id.$key.output_falco_*.output_falco_*" + "default":"$id.$key.output_falco_*" } diff --git a/target/nextflow/dataflow/gather_fastqs_and_validate/.config.vsh.yaml b/target/nextflow/dataflow/gather_fastqs_and_validate/.config.vsh.yaml index 7ca1b6d..577ac29 100644 --- a/target/nextflow/dataflow/gather_fastqs_and_validate/.config.vsh.yaml +++ b/target/nextflow/dataflow/gather_fastqs_and_validate/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "gather_fastqs_and_validate" namespace: "dataflow" -version: "v0.3.8" +version: "v0.3.9" argument_groups: - name: "Input arguments" arguments: @@ -56,6 +56,9 @@ description: "From a directory containing fastq files, gather the files per samp \ \nand validate according to the contents of the sample sheet.\n" info: null status: "enabled" +scope: + image: "public" + target: "public" requirements: commands: - "ps" @@ -137,29 +140,29 @@ build_info: engine: "native|native" output: "target/nextflow/dataflow/gather_fastqs_and_validate" executable: "target/nextflow/dataflow/gather_fastqs_and_validate/main.nf" - viash_version: "0.9.0" - git_commit: "dafe2d829079bd5a9f5eb4e7cf63e92edbf1a742" + viash_version: "0.9.4" + git_commit: "820318c378d8119e2aa98768282e43c2aa017ba7" git_remote: "https://github.com/viash-hub/demultiplex" - git_tag: "v0.3.5-9-gdafe2d8" + git_tag: "v0.3.8-5-g820318c" package_config: name: "demultiplex" - version: "v0.3.8" + version: "v0.3.9" description: "Demultiplexing pipeline\n" info: test_resources: - path: "gs://viash-hub-test-data/demultiplex/v2/" dest: "testData" - viash_version: "0.9.0" + viash_version: "0.9.4" source: "src" target: "target" config_mods: - - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\ + - ".requirements.commands += ['ps']\n.runners[.type == 'nextflow'].directives.tag\ \ := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n\ .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ )'\n" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.3.8'" + - ".engines[.type == 'docker'].target_tag := 'v0.3.9'" keywords: - "bioinformatics" - "sequence" diff --git a/target/nextflow/dataflow/gather_fastqs_and_validate/main.nf b/target/nextflow/dataflow/gather_fastqs_and_validate/main.nf index 33c2770..13d19f7 100644 --- a/target/nextflow/dataflow/gather_fastqs_and_validate/main.nf +++ b/target/nextflow/dataflow/gather_fastqs_and_validate/main.nf @@ -1,6 +1,6 @@ -// gather_fastqs_and_validate v0.3.8 +// gather_fastqs_and_validate v0.3.9 // -// This wrapper script is auto-generated by viash 0.9.0 and is thus a derivative +// This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data // Intuitive. // @@ -82,64 +82,56 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi foundClass = "List[${e.foundClass}]" } } else if (par.type == "string") { - // cast to string if need be + // cast to string if need be. only cast if the value is a GString if (value instanceof GString) { - value = value.toString() + value = value as String } expectedClass = value instanceof String ? null : "String" } else if (par.type == "integer") { // cast to integer if need be - if (value instanceof String) { + if (value !instanceof Integer) { try { - value = value.toInteger() + value = value as Integer } catch (NumberFormatException e) { - // do nothing + expectedClass = "Integer" } } - if (value instanceof java.math.BigInteger) { - value = value.intValue() - } - expectedClass = value instanceof Integer ? null : "Integer" } else if (par.type == "long") { // cast to long if need be - if (value instanceof String) { + if (value !instanceof Long) { try { - value = value.toLong() + value = value as Long } catch (NumberFormatException e) { - // do nothing + expectedClass = "Long" } } - if (value instanceof Integer) { - value = value.toLong() - } - expectedClass = value instanceof Long ? null : "Long" } else if (par.type == "double") { // cast to double if need be - if (value instanceof String) { + if (value !instanceof Double) { try { - value = value.toDouble() + value = value as Double } catch (NumberFormatException e) { - // do nothing + expectedClass = "Double" } } - if (value instanceof java.math.BigDecimal) { - value = value.doubleValue() + } else if (par.type == "float") { + // cast to float if need be + if (value !instanceof Float) { + try { + value = value as Float + } catch (NumberFormatException e) { + expectedClass = "Float" + } } - if (value instanceof Float) { - value = value.toDouble() - } - expectedClass = value instanceof Double ? null : "Double" } else if (par.type == "boolean" | par.type == "boolean_true" | par.type == "boolean_false") { // cast to boolean if need be - if (value instanceof String) { - def valueLower = value.toLowerCase() - if (valueLower == "true") { - value = true - } else if (valueLower == "false") { - value = false + if (value !instanceof Boolean) { + try { + value = value as Boolean + } catch (Exception e) { + expectedClass = "Boolean" } } - expectedClass = value instanceof Boolean ? null : "Boolean" } else if (par.type == "file" && (par.direction == "input" || stage == "output")) { // cast to path if need be if (value instanceof String) { @@ -151,10 +143,13 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi expectedClass = value instanceof Path ? null : "Path" } else if (par.type == "file" && stage == "input" && par.direction == "output") { // cast to string if need be - if (value instanceof GString) { - value = value.toString() + if (value !instanceof String) { + try { + value = value as String + } catch (Exception e) { + expectedClass = "String" + } } - expectedClass = value instanceof String ? null : "String" } else { // didn't find a match for par.type expectedClass = par.type @@ -173,7 +168,7 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi Map _processInputValues(Map inputs, Map config, String id, String key) { if (!workflow.stubRun) { config.allArguments.each { arg -> - if (arg.required) { + if (arg.required && arg.direction == "input") { assert inputs.containsKey(arg.plainName) && inputs.get(arg.plainName) != null : "Error in module '${key}' id '${id}': required input argument '${arg.plainName}' is missing" } @@ -192,15 +187,8 @@ Map _processInputValues(Map inputs, Map config, String id, String key) { } // helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_processOutputValues.nf' -Map _processOutputValues(Map outputs, Map config, String id, String key) { +Map _checkValidOutputArgument(Map outputs, Map config, String id, String key) { if (!workflow.stubRun) { - config.allArguments.each { arg -> - if (arg.direction == "output" && arg.required) { - assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : - "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing" - } - } - outputs = outputs.collectEntries { name, value -> def par = config.allArguments.find { it.plainName == name && it.direction == "output" } assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid output argument" @@ -213,6 +201,16 @@ Map _processOutputValues(Map outputs, Map config, String id, String key) { return outputs } +void _checkAllRequiredOuputsPresent(Map outputs, Map config, String id, String key) { + if (!workflow.stubRun) { + config.allArguments.each { arg -> + if (arg.direction == "output" && arg.required) { + assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : + "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing" + } + } + } +} // helper file: 'src/main/resources/io/viash/runners/nextflow/channel/IDChecker.nf' class IDChecker { final def items = [] as Set @@ -1666,6 +1664,162 @@ def joinStates(Closure apply_) { } return joinStatesWf } +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishFiles.nf' +def publishFiles(Map args) { + def key_ = args.get("key") + + assert key_ != null : "publishFiles: key must be specified" + + workflow publishFilesWf { + take: input_ch + main: + input_ch + | map { tup -> + def id_ = tup[0] + def state_ = tup[1] + + // the input files and the target output filenames + def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose() + def inputFiles_ = inputoutputFilenames_[0] + def outputFilenames_ = inputoutputFilenames_[1] + + [id_, inputFiles_, outputFilenames_] + } + | publishFilesProc + emit: input_ch + } + return publishFilesWf +} + +process publishFilesProc { + // todo: check publishpath? + publishDir path: "${getPublishDir()}/", mode: "copy" + tag "$id" + input: + tuple val(id), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles) + output: + tuple val(id), path{outputFiles} + script: + def copyCommands = [ + inputFiles instanceof List ? inputFiles : [inputFiles], + outputFiles instanceof List ? outputFiles : [outputFiles] + ] + .transpose() + .collectMany{infile, outfile -> + if (infile.toString() != outfile.toString()) { + [ + "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"", + "cp -r '${infile.toString()}' '${outfile.toString()}'" + ] + } else { + // no need to copy if infile is the same as outfile + [] + } + } + """ + echo "Copying output files to destination folder" + ${copyCommands.join("\n ")} + """ +} + + +// this assumes that the state contains no other values other than those specified in the config +def publishFilesByConfig(Map args) { + def config = args.get("config") + assert config != null : "publishFilesByConfig: config must be specified" + + def key_ = args.get("key", config.name) + assert key_ != null : "publishFilesByConfig: key must be specified" + + workflow publishFilesSimpleWf { + take: input_ch + main: + input_ch + | map { tup -> + def id_ = tup[0] + def state_ = tup[1] // e.g. [output: new File("myoutput.h5ad"), k: 10] + def origState_ = tup[2] // e.g. [output: '$id.$key.foo.h5ad'] + + + // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where + // - key is a String + // - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path) + // - inputPath is a List[Path] + // - outputFilename is a List[String] + // - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml) + def processedState = + config.allArguments + .findAll { it.direction == "output" } + .collectMany { par -> + def plainName_ = par.plainName + // if the state does not contain the key, it's an + // optional argument for which the component did + // not generate any output OR multiple channels were emitted + // and the output was just not added to using the channel + // that is now being parsed + if (!state_.containsKey(plainName_)) { + return [] + } + def value = state_[plainName_] + // if the parameter is not a file, it should be stored + // in the state as-is, but is not something that needs + // to be copied from the source path to the dest path + if (par.type != "file") { + return [[inputPath: [], outputFilename: []]] + } + // if the orig state does not contain this filename, + // it's an optional argument for which the user specified + // that it should not be returned as a state + if (!origState_.containsKey(plainName_)) { + return [] + } + def filenameTemplate = origState_[plainName_] + // if the pararameter is multiple: true, fetch the template + if (par.multiple && filenameTemplate instanceof List) { + filenameTemplate = filenameTemplate[0] + } + // instantiate the template + def filename = filenameTemplate + .replaceAll('\\$id', id_) + .replaceAll('\\$\\{id\\}', id_) + .replaceAll('\\$key', key_) + .replaceAll('\\$\\{key\\}', key_) + if (par.multiple) { + // if the parameter is multiple: true, the filename + // should contain a wildcard '*' that is replaced with + // the index of the file + assert filename.contains("*") : "Module '${key_}' id '${id_}': Multiple output files specified, but no wildcard '*' in the filename: ${filename}" + def outputPerFile = value.withIndex().collect{ val, ix -> + def filename_ix = filename.replace("*", ix.toString()) + def inputPath = val instanceof File ? val.toPath() : val + [inputPath: inputPath, outputFilename: filename_ix] + } + def transposedOutputs = ["inputPath", "outputFilename"].collectEntries{ key -> + [key, outputPerFile.collect{dic -> dic[key]}] + } + return [[key: plainName_] + transposedOutputs] + } else { + def value_ = java.nio.file.Paths.get(filename) + def inputPath = value instanceof File ? value.toPath() : value + return [[inputPath: [inputPath], outputFilename: [filename]]] + } + } + + def inputPaths = processedState.collectMany{it.inputPath} + def outputFilenames = processedState.collectMany{it.outputFilename} + + + [id_, inputPaths, outputFilenames] + } + | publishFilesProc + emit: input_ch + } + return publishFilesSimpleWf +} + + + + // helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishStates.nf' def collectFiles(obj) { if (obj instanceof java.io.File || obj instanceof Path) { @@ -1723,8 +1877,6 @@ def publishStates(Map args) { // the input files and the target output filenames def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose() - def inputFiles_ = inputoutputFilenames_[0] - def outputFilenames_ = inputoutputFilenames_[1] def yamlFilename = yamlTemplate_ .replaceAll('\\$id', id_) @@ -1737,7 +1889,7 @@ def publishStates(Map args) { // convert state to yaml blob def yamlBlob_ = toRelativeTaggedYamlBlob([id: id_] + state_, java.nio.file.Paths.get(yamlFilename)) - [id_, yamlBlob_, yamlFilename, inputFiles_, outputFilenames_] + [id_, yamlBlob_, yamlFilename] } | publishStatesProc emit: input_ch @@ -1749,33 +1901,17 @@ process publishStatesProc { publishDir path: "${getPublishDir()}/", mode: "copy" tag "$id" input: - tuple val(id), val(yamlBlob), val(yamlFile), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles) + tuple val(id), val(yamlBlob), val(yamlFile) output: - tuple val(id), path{[yamlFile] + outputFiles} + tuple val(id), path{[yamlFile]} script: - def copyCommands = [ - inputFiles instanceof List ? inputFiles : [inputFiles], - outputFiles instanceof List ? outputFiles : [outputFiles] - ] - .transpose() - .collectMany{infile, outfile -> - if (infile.toString() != outfile.toString()) { - [ - "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"", - "cp -r '${infile.toString()}' '${outfile.toString()}'" - ] - } else { - // no need to copy if infile is the same as outfile - [] - } - } """ -mkdir -p "\$(dirname '${yamlFile}')" -echo "Storing state as yaml" -echo '${yamlBlob}' > '${yamlFile}' -echo "Copying output files to destination folder" -${copyCommands.join("\n ")} -""" + mkdir -p "\$(dirname '${yamlFile}')" + echo "Storing state as yaml" + cat > '${yamlFile}' << HERE +${yamlBlob} +HERE + """ } @@ -1806,13 +1942,10 @@ def publishStatesByConfig(Map args) { .replaceAll('\\$\\{key\\}', key_) def yamlDir = java.nio.file.Paths.get(yamlFilename).getParent() - // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where + // the processed state is a list of [key, value] tuples, where // - key is a String // - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path) - // - inputPath is a List[Path] - // - outputFilename is a List[String] // - (key, value) are the tuples that will be saved to the state.yaml file - // - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml) def processedState = config.allArguments .findAll { it.direction == "output" } @@ -1829,7 +1962,7 @@ def publishStatesByConfig(Map args) { // in the state as-is, but is not something that needs // to be copied from the source path to the dest path if (par.type != "file") { - return [[key: plainName_, value: value, inputPath: [], outputFilename: []]] + return [[key: plainName_, value: value]] } // if the orig state does not contain this filename, // it's an optional argument for which the user specified @@ -1860,13 +1993,9 @@ def publishStatesByConfig(Map args) { if (yamlDir != null) { value_ = yamlDir.relativize(value_) } - def inputPath = val instanceof File ? val.toPath() : val - [value: value_, inputPath: inputPath, outputFilename: filename_ix] + return value_ } - def transposedOutputs = ["value", "inputPath", "outputFilename"].collectEntries{ key -> - [key, outputPerFile.collect{dic -> dic[key]}] - } - return [[key: plainName_] + transposedOutputs] + return [["key": plainName_, "value": outputPerFile]] } else { def value_ = java.nio.file.Paths.get(filename) // if id contains a slash @@ -1874,18 +2003,17 @@ def publishStatesByConfig(Map args) { value_ = yamlDir.relativize(value_) } def inputPath = value instanceof File ? value.toPath() : value - return [[key: plainName_, value: value_, inputPath: [inputPath], outputFilename: [filename]]] + return [["key": plainName_, value: value_]] } } + def updatedState_ = processedState.collectEntries{[it.key, it.value]} - def inputPaths = processedState.collectMany{it.inputPath} - def outputFilenames = processedState.collectMany{it.outputFilename} // convert state to yaml blob def yamlBlob_ = toTaggedYamlBlob([id: id_] + updatedState_) - [id_, yamlBlob_, yamlFilename, inputPaths, outputFilenames] + [id_, yamlBlob_, yamlFilename] } | publishStatesProc emit: input_ch @@ -2559,7 +2687,8 @@ def _debug(workflowArgs, debugKey) { def workflowFactory(Map args, Map defaultWfArgs, Map meta) { def workflowArgs = processWorkflowArgs(args, defaultWfArgs, meta) def key_ = workflowArgs["key"] - + def multipleArgs = meta.config.allArguments.findAll{ it.multiple }.collect{it.plainName} + workflow workflowInstance { take: input_ @@ -2716,12 +2845,36 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { } // TODO: move some of the _meta.join_id wrangling to the safeJoin() function. - def chInitialOutput = chArgsWithDefaults + def chInitialOutputMulti = chArgsWithDefaults | _debug(workflowArgs, "processed") // run workflow | innerWorkflowFactory(workflowArgs) - // check output tuple - | map { id_, output_ -> + def chInitialOutputList = chInitialOutputMulti instanceof List ? chInitialOutputMulti : [chInitialOutputMulti] + assert chInitialOutputList.size() > 0: "should have emitted at least one output channel" + // Add a channel ID to the events, which designates the channel the event was emitted from as a running number + // This number is used to sort the events later when the events are gathered from across the channels. + def chInitialOutputListWithIndexedEvents = chInitialOutputList.withIndex().collect{channel, channelIndex -> + def newChannel = channel + | map {tuple -> + assert tuple instanceof List : + "Error in module '${key_}': element in output channel should be a tuple [id, data, ...otherargs...]\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Expected class: List. Found: tuple.getClass() is ${tuple.getClass()}" + + def newEvent = [channelIndex] + tuple + return newEvent + } + return newChannel + } + // Put the events into 1 channel, cover case where there is only one channel is emitted + def chInitialOutput = chInitialOutputList.size() > 1 ? \ + chInitialOutputListWithIndexedEvents[0].mix(*chInitialOutputListWithIndexedEvents.tail()) : \ + chInitialOutputListWithIndexedEvents[0] + def chInitialOutputProcessed = chInitialOutput + | map { tuple -> + def channelId = tuple[0] + def id_ = tuple[1] + def output_ = tuple[2] // see if output map contains metadata def meta_ = @@ -2734,19 +2887,94 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { output_ = output_.findAll{k, v -> k != "_meta"} // check value types - output_ = _processOutputValues(output_, meta.config, id_, key_) + output_ = _checkValidOutputArgument(output_, meta.config, id_, key_) - // simplify output if need be - if (workflowArgs.auto.simplifyOutput && output_.size() == 1) { - output_ = output_.values()[0] - } - - [join_id, id_, output_] + [join_id, channelId, id_, output_] } // | view{"chInitialOutput: ${it.take(3)}"} + // join the output [prev_id, channel_id, new_id, output] with the previous state [prev_id, state, ...] + def chPublishWithPreviousState = safeJoin(chInitialOutputProcessed, chRunFiltered, key_) + // input tuple format: [join_id, channel_id, id, output, prev_state, ...] + // output tuple format: [join_id, channel_id, id, new_state, ...] + | map{ tup -> + def new_state = workflowArgs.toState(tup.drop(2).take(3)) + tup.take(3) + [new_state] + tup.drop(5) + } + if (workflowArgs.auto.publish == "state") { + def chPublishFiles = chPublishWithPreviousState + // input tuple format: [join_id, channel_id, id, new_state, ...] + // output tuple format: [join_id, channel_id, id, new_state] + | map{ tup -> + tup.take(4) + } + + safeJoin(chPublishFiles, chArgsWithDefaults, key_) + // input tuple format: [join_id, channel_id, id, new_state, orig_state, ...] + // output tuple format: [id, new_state, orig_state] + | map { tup -> + tup.drop(2).take(3) + } + | publishFilesByConfig(key: key_, config: meta.config) + } + // Join the state from the events that were emitted from different channels + def chJoined = chInitialOutputProcessed + | map {tuple -> + def join_id = tuple[0] + def channel_id = tuple[1] + def id = tuple[2] + def other = tuple.drop(3) + // Below, groupTuple is used to join the events. To make sure resuming a workflow + // keeps working, the output state must be deterministic. This means the state needs to be + // sorted with groupTuple's has a 'sort' argument. This argument can be set to 'hash', + // but hashing the state when it is large can be problematic in terms of performance. + // Therefore, a custom comparator function is provided. We add the channel ID to the + // states so that we can use the channel ID to sort the items. + def stateWithChannelID = [[channel_id] * other.size(), other].transpose() + // A comparator that is provided to groupTuple's 'sort' argument is applied + // to all elements of the event tuple (that is not the 'id'). The comparator + // closure that is used below expects the input to be List. So the join_id and + // channel_id must also be wrapped in a list. + [[join_id], [channel_id], id] + stateWithChannelID + } + | groupTuple(by: 2, sort: {a, b -> a[0] <=> b[0]}, size: chInitialOutputList.size(), remainder: true) + | map {join_ids, _, id, statesWithChannelID -> + // Remove the channel IDs from the states + def states = statesWithChannelID.collect{it[1]} + def newJoinId = join_ids.flatten().unique{a, b -> a <=> b} + assert newJoinId.size() == 1: "Multiple events were emitted for '$id'." + def newJoinIdUnique = newJoinId[0] + + // Merge the states from the different channels + def newState = states.inject([:]){ old_state, state_to_add -> + return old_state + state_to_add.collectEntries{k, v -> + if (!multipleArgs.contains(k)) { + // if the key is not a multiple argument, we expect only one value + if (old_state.containsKey(k)) { + assert old_state[k] == v : "ID $id: multiple entries for argument $k were emitted." + } + [k, v] + } else { + // if the key is a multiple argument, append the different values into one list + def prevValue = old_state.getOrDefault(k, []) + def prevValueAsList = prevValue instanceof List ? prevValue : [prevValue] + [k, prevValueAsList + v] + } + } + } + + _checkAllRequiredOuputsPresent(newState, meta.config, id, key_) + + // simplify output if need be + if (workflowArgs.auto.simplifyOutput && newState.size() == 1) { + newState = newState.values()[0] + } + + return [newJoinIdUnique, id, newState] + } + // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) + def chNewState = safeJoin(chJoined, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -2755,23 +2983,21 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { } if (workflowArgs.auto.publish == "state") { - def chPublish = chNewState + def chPublishStates = chNewState // input tuple format: [join_id, id, new_state, ...] // output tuple format: [join_id, id, new_state] | map{ tup -> tup.take(3) } - safeJoin(chPublish, chArgsWithDefaults, key_) + safeJoin(chPublishStates, chArgsWithDefaults, key_) // input tuple format: [join_id, id, new_state, orig_state, ...] // output tuple format: [id, new_state, orig_state] | map { tup -> tup.drop(1).take(3) - } + } | publishStatesByConfig(key: key_, config: meta.config) } - - // remove join_id and meta chReturn = chNewState | map { tup -> // input tuple format: [join_id, id, new_state, ...] @@ -2806,7 +3032,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "gather_fastqs_and_validate", "namespace" : "dataflow", - "version" : "v0.3.8", + "version" : "v0.3.9", "argument_groups" : [ { "name" : "Input arguments", @@ -2876,6 +3102,10 @@ meta = [ ], "description" : "From a directory containing fastq files, gather the files per sample \nand validate according to the contents of the sample sheet.\n", "status" : "enabled", + "scope" : { + "image" : "public", + "target" : "public" + }, "requirements" : { "commands" : [ "ps" @@ -2972,14 +3202,14 @@ meta = [ "runner" : "nextflow", "engine" : "native|native", "output" : "target/nextflow/dataflow/gather_fastqs_and_validate", - "viash_version" : "0.9.0", - "git_commit" : "dafe2d829079bd5a9f5eb4e7cf63e92edbf1a742", + "viash_version" : "0.9.4", + "git_commit" : "820318c378d8119e2aa98768282e43c2aa017ba7", "git_remote" : "https://github.com/viash-hub/demultiplex", - "git_tag" : "v0.3.5-9-gdafe2d8" + "git_tag" : "v0.3.8-5-g820318c" }, "package_config" : { "name" : "demultiplex", - "version" : "v0.3.8", + "version" : "v0.3.9", "description" : "Demultiplexing pipeline\n", "info" : { "test_resources" : [ @@ -2989,14 +3219,14 @@ meta = [ } ] }, - "viash_version" : "0.9.0", + "viash_version" : "0.9.4", "source" : "src", "target" : "target", "config_mods" : [ - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n", + ".requirements.commands += ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v0.3.8'" + ".engines[.type == 'docker'].target_tag := 'v0.3.9'" ], "keywords" : [ "bioinformatics", diff --git a/target/nextflow/dataflow/gather_fastqs_and_validate/nextflow.config b/target/nextflow/dataflow/gather_fastqs_and_validate/nextflow.config index 1828dda..05365ca 100644 --- a/target/nextflow/dataflow/gather_fastqs_and_validate/nextflow.config +++ b/target/nextflow/dataflow/gather_fastqs_and_validate/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'dataflow/gather_fastqs_and_validate' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v0.3.8' + version = 'v0.3.9' description = 'From a directory containing fastq files, gather the files per sample \nand validate according to the contents of the sample sheet.\n' } diff --git a/target/nextflow/dataflow/gather_fastqs_and_validate/nextflow_schema.json b/target/nextflow/dataflow/gather_fastqs_and_validate/nextflow_schema.json index 311299a..aac635e 100644 --- a/target/nextflow/dataflow/gather_fastqs_and_validate/nextflow_schema.json +++ b/target/nextflow/dataflow/gather_fastqs_and_validate/nextflow_schema.json @@ -47,10 +47,10 @@ "fastq_forward": { "type": "string", - "description": "Type: List of `file`, required, default: `$id.$key.fastq_forward_*.fastq_forward_*`, multiple_sep: `\";\"`. ", - "help_text": "Type: List of `file`, required, default: `$id.$key.fastq_forward_*.fastq_forward_*`, multiple_sep: `\";\"`. " + "description": "Type: List of `file`, required, default: `$id.$key.fastq_forward_*`, multiple_sep: `\";\"`. ", + "help_text": "Type: List of `file`, required, default: `$id.$key.fastq_forward_*`, multiple_sep: `\";\"`. " , - "default":"$id.$key.fastq_forward_*.fastq_forward_*" + "default":"$id.$key.fastq_forward_*" } @@ -58,10 +58,10 @@ "fastq_reverse": { "type": "string", - "description": "Type: List of `file`, default: `$id.$key.fastq_reverse_*.fastq_reverse_*`, multiple_sep: `\";\"`. ", - "help_text": "Type: List of `file`, default: `$id.$key.fastq_reverse_*.fastq_reverse_*`, multiple_sep: `\";\"`. " + "description": "Type: List of `file`, default: `$id.$key.fastq_reverse_*`, multiple_sep: `\";\"`. ", + "help_text": "Type: List of `file`, default: `$id.$key.fastq_reverse_*`, multiple_sep: `\";\"`. " , - "default":"$id.$key.fastq_reverse_*.fastq_reverse_*" + "default":"$id.$key.fastq_reverse_*" } diff --git a/target/nextflow/demultiplex/.config.vsh.yaml b/target/nextflow/demultiplex/.config.vsh.yaml index 20d38a8..2664d03 100644 --- a/target/nextflow/demultiplex/.config.vsh.yaml +++ b/target/nextflow/demultiplex/.config.vsh.yaml @@ -1,5 +1,5 @@ name: "demultiplex" -version: "v0.3.8" +version: "v0.3.9" argument_groups: - name: "Input arguments" arguments: @@ -124,6 +124,9 @@ test_resources: entrypoint: "test_bases2fastq" info: null status: "enabled" +scope: + image: "public" + target: "public" requirements: commands: - "ps" @@ -243,10 +246,10 @@ build_info: engine: "native|native" output: "target/nextflow/demultiplex" executable: "target/nextflow/demultiplex/main.nf" - viash_version: "0.9.0" - git_commit: "dafe2d829079bd5a9f5eb4e7cf63e92edbf1a742" + viash_version: "0.9.4" + git_commit: "820318c378d8119e2aa98768282e43c2aa017ba7" git_remote: "https://github.com/viash-hub/demultiplex" - git_tag: "v0.3.5-9-gdafe2d8" + git_tag: "v0.3.8-5-g820318c" dependencies: - "target/nextflow/io/untar" - "target/nextflow/dataflow/gather_fastqs_and_validate" @@ -258,23 +261,23 @@ build_info: - "target/dependencies/vsh/vsh/biobox/v0.3.0/nextflow/multiqc" package_config: name: "demultiplex" - version: "v0.3.8" + version: "v0.3.9" description: "Demultiplexing pipeline\n" info: test_resources: - path: "gs://viash-hub-test-data/demultiplex/v2/" dest: "testData" - viash_version: "0.9.0" + viash_version: "0.9.4" source: "src" target: "target" config_mods: - - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\ + - ".requirements.commands += ['ps']\n.runners[.type == 'nextflow'].directives.tag\ \ := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n\ .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ )'\n" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.3.8'" + - ".engines[.type == 'docker'].target_tag := 'v0.3.9'" keywords: - "bioinformatics" - "sequence" diff --git a/target/nextflow/demultiplex/main.nf b/target/nextflow/demultiplex/main.nf index 0b1cc67..3a508b1 100644 --- a/target/nextflow/demultiplex/main.nf +++ b/target/nextflow/demultiplex/main.nf @@ -1,6 +1,6 @@ -// demultiplex v0.3.8 +// demultiplex v0.3.9 // -// This wrapper script is auto-generated by viash 0.9.0 and is thus a derivative +// This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data // Intuitive. // @@ -82,64 +82,56 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi foundClass = "List[${e.foundClass}]" } } else if (par.type == "string") { - // cast to string if need be + // cast to string if need be. only cast if the value is a GString if (value instanceof GString) { - value = value.toString() + value = value as String } expectedClass = value instanceof String ? null : "String" } else if (par.type == "integer") { // cast to integer if need be - if (value instanceof String) { + if (value !instanceof Integer) { try { - value = value.toInteger() + value = value as Integer } catch (NumberFormatException e) { - // do nothing + expectedClass = "Integer" } } - if (value instanceof java.math.BigInteger) { - value = value.intValue() - } - expectedClass = value instanceof Integer ? null : "Integer" } else if (par.type == "long") { // cast to long if need be - if (value instanceof String) { + if (value !instanceof Long) { try { - value = value.toLong() + value = value as Long } catch (NumberFormatException e) { - // do nothing + expectedClass = "Long" } } - if (value instanceof Integer) { - value = value.toLong() - } - expectedClass = value instanceof Long ? null : "Long" } else if (par.type == "double") { // cast to double if need be - if (value instanceof String) { + if (value !instanceof Double) { try { - value = value.toDouble() + value = value as Double } catch (NumberFormatException e) { - // do nothing + expectedClass = "Double" } } - if (value instanceof java.math.BigDecimal) { - value = value.doubleValue() + } else if (par.type == "float") { + // cast to float if need be + if (value !instanceof Float) { + try { + value = value as Float + } catch (NumberFormatException e) { + expectedClass = "Float" + } } - if (value instanceof Float) { - value = value.toDouble() - } - expectedClass = value instanceof Double ? null : "Double" } else if (par.type == "boolean" | par.type == "boolean_true" | par.type == "boolean_false") { // cast to boolean if need be - if (value instanceof String) { - def valueLower = value.toLowerCase() - if (valueLower == "true") { - value = true - } else if (valueLower == "false") { - value = false + if (value !instanceof Boolean) { + try { + value = value as Boolean + } catch (Exception e) { + expectedClass = "Boolean" } } - expectedClass = value instanceof Boolean ? null : "Boolean" } else if (par.type == "file" && (par.direction == "input" || stage == "output")) { // cast to path if need be if (value instanceof String) { @@ -151,10 +143,13 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi expectedClass = value instanceof Path ? null : "Path" } else if (par.type == "file" && stage == "input" && par.direction == "output") { // cast to string if need be - if (value instanceof GString) { - value = value.toString() + if (value !instanceof String) { + try { + value = value as String + } catch (Exception e) { + expectedClass = "String" + } } - expectedClass = value instanceof String ? null : "String" } else { // didn't find a match for par.type expectedClass = par.type @@ -173,7 +168,7 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi Map _processInputValues(Map inputs, Map config, String id, String key) { if (!workflow.stubRun) { config.allArguments.each { arg -> - if (arg.required) { + if (arg.required && arg.direction == "input") { assert inputs.containsKey(arg.plainName) && inputs.get(arg.plainName) != null : "Error in module '${key}' id '${id}': required input argument '${arg.plainName}' is missing" } @@ -192,15 +187,8 @@ Map _processInputValues(Map inputs, Map config, String id, String key) { } // helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_processOutputValues.nf' -Map _processOutputValues(Map outputs, Map config, String id, String key) { +Map _checkValidOutputArgument(Map outputs, Map config, String id, String key) { if (!workflow.stubRun) { - config.allArguments.each { arg -> - if (arg.direction == "output" && arg.required) { - assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : - "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing" - } - } - outputs = outputs.collectEntries { name, value -> def par = config.allArguments.find { it.plainName == name && it.direction == "output" } assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid output argument" @@ -213,6 +201,16 @@ Map _processOutputValues(Map outputs, Map config, String id, String key) { return outputs } +void _checkAllRequiredOuputsPresent(Map outputs, Map config, String id, String key) { + if (!workflow.stubRun) { + config.allArguments.each { arg -> + if (arg.direction == "output" && arg.required) { + assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : + "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing" + } + } + } +} // helper file: 'src/main/resources/io/viash/runners/nextflow/channel/IDChecker.nf' class IDChecker { final def items = [] as Set @@ -1666,6 +1664,162 @@ def joinStates(Closure apply_) { } return joinStatesWf } +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishFiles.nf' +def publishFiles(Map args) { + def key_ = args.get("key") + + assert key_ != null : "publishFiles: key must be specified" + + workflow publishFilesWf { + take: input_ch + main: + input_ch + | map { tup -> + def id_ = tup[0] + def state_ = tup[1] + + // the input files and the target output filenames + def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose() + def inputFiles_ = inputoutputFilenames_[0] + def outputFilenames_ = inputoutputFilenames_[1] + + [id_, inputFiles_, outputFilenames_] + } + | publishFilesProc + emit: input_ch + } + return publishFilesWf +} + +process publishFilesProc { + // todo: check publishpath? + publishDir path: "${getPublishDir()}/", mode: "copy" + tag "$id" + input: + tuple val(id), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles) + output: + tuple val(id), path{outputFiles} + script: + def copyCommands = [ + inputFiles instanceof List ? inputFiles : [inputFiles], + outputFiles instanceof List ? outputFiles : [outputFiles] + ] + .transpose() + .collectMany{infile, outfile -> + if (infile.toString() != outfile.toString()) { + [ + "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"", + "cp -r '${infile.toString()}' '${outfile.toString()}'" + ] + } else { + // no need to copy if infile is the same as outfile + [] + } + } + """ + echo "Copying output files to destination folder" + ${copyCommands.join("\n ")} + """ +} + + +// this assumes that the state contains no other values other than those specified in the config +def publishFilesByConfig(Map args) { + def config = args.get("config") + assert config != null : "publishFilesByConfig: config must be specified" + + def key_ = args.get("key", config.name) + assert key_ != null : "publishFilesByConfig: key must be specified" + + workflow publishFilesSimpleWf { + take: input_ch + main: + input_ch + | map { tup -> + def id_ = tup[0] + def state_ = tup[1] // e.g. [output: new File("myoutput.h5ad"), k: 10] + def origState_ = tup[2] // e.g. [output: '$id.$key.foo.h5ad'] + + + // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where + // - key is a String + // - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path) + // - inputPath is a List[Path] + // - outputFilename is a List[String] + // - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml) + def processedState = + config.allArguments + .findAll { it.direction == "output" } + .collectMany { par -> + def plainName_ = par.plainName + // if the state does not contain the key, it's an + // optional argument for which the component did + // not generate any output OR multiple channels were emitted + // and the output was just not added to using the channel + // that is now being parsed + if (!state_.containsKey(plainName_)) { + return [] + } + def value = state_[plainName_] + // if the parameter is not a file, it should be stored + // in the state as-is, but is not something that needs + // to be copied from the source path to the dest path + if (par.type != "file") { + return [[inputPath: [], outputFilename: []]] + } + // if the orig state does not contain this filename, + // it's an optional argument for which the user specified + // that it should not be returned as a state + if (!origState_.containsKey(plainName_)) { + return [] + } + def filenameTemplate = origState_[plainName_] + // if the pararameter is multiple: true, fetch the template + if (par.multiple && filenameTemplate instanceof List) { + filenameTemplate = filenameTemplate[0] + } + // instantiate the template + def filename = filenameTemplate + .replaceAll('\\$id', id_) + .replaceAll('\\$\\{id\\}', id_) + .replaceAll('\\$key', key_) + .replaceAll('\\$\\{key\\}', key_) + if (par.multiple) { + // if the parameter is multiple: true, the filename + // should contain a wildcard '*' that is replaced with + // the index of the file + assert filename.contains("*") : "Module '${key_}' id '${id_}': Multiple output files specified, but no wildcard '*' in the filename: ${filename}" + def outputPerFile = value.withIndex().collect{ val, ix -> + def filename_ix = filename.replace("*", ix.toString()) + def inputPath = val instanceof File ? val.toPath() : val + [inputPath: inputPath, outputFilename: filename_ix] + } + def transposedOutputs = ["inputPath", "outputFilename"].collectEntries{ key -> + [key, outputPerFile.collect{dic -> dic[key]}] + } + return [[key: plainName_] + transposedOutputs] + } else { + def value_ = java.nio.file.Paths.get(filename) + def inputPath = value instanceof File ? value.toPath() : value + return [[inputPath: [inputPath], outputFilename: [filename]]] + } + } + + def inputPaths = processedState.collectMany{it.inputPath} + def outputFilenames = processedState.collectMany{it.outputFilename} + + + [id_, inputPaths, outputFilenames] + } + | publishFilesProc + emit: input_ch + } + return publishFilesSimpleWf +} + + + + // helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishStates.nf' def collectFiles(obj) { if (obj instanceof java.io.File || obj instanceof Path) { @@ -1723,8 +1877,6 @@ def publishStates(Map args) { // the input files and the target output filenames def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose() - def inputFiles_ = inputoutputFilenames_[0] - def outputFilenames_ = inputoutputFilenames_[1] def yamlFilename = yamlTemplate_ .replaceAll('\\$id', id_) @@ -1737,7 +1889,7 @@ def publishStates(Map args) { // convert state to yaml blob def yamlBlob_ = toRelativeTaggedYamlBlob([id: id_] + state_, java.nio.file.Paths.get(yamlFilename)) - [id_, yamlBlob_, yamlFilename, inputFiles_, outputFilenames_] + [id_, yamlBlob_, yamlFilename] } | publishStatesProc emit: input_ch @@ -1749,33 +1901,17 @@ process publishStatesProc { publishDir path: "${getPublishDir()}/", mode: "copy" tag "$id" input: - tuple val(id), val(yamlBlob), val(yamlFile), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles) + tuple val(id), val(yamlBlob), val(yamlFile) output: - tuple val(id), path{[yamlFile] + outputFiles} + tuple val(id), path{[yamlFile]} script: - def copyCommands = [ - inputFiles instanceof List ? inputFiles : [inputFiles], - outputFiles instanceof List ? outputFiles : [outputFiles] - ] - .transpose() - .collectMany{infile, outfile -> - if (infile.toString() != outfile.toString()) { - [ - "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"", - "cp -r '${infile.toString()}' '${outfile.toString()}'" - ] - } else { - // no need to copy if infile is the same as outfile - [] - } - } """ -mkdir -p "\$(dirname '${yamlFile}')" -echo "Storing state as yaml" -echo '${yamlBlob}' > '${yamlFile}' -echo "Copying output files to destination folder" -${copyCommands.join("\n ")} -""" + mkdir -p "\$(dirname '${yamlFile}')" + echo "Storing state as yaml" + cat > '${yamlFile}' << HERE +${yamlBlob} +HERE + """ } @@ -1806,13 +1942,10 @@ def publishStatesByConfig(Map args) { .replaceAll('\\$\\{key\\}', key_) def yamlDir = java.nio.file.Paths.get(yamlFilename).getParent() - // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where + // the processed state is a list of [key, value] tuples, where // - key is a String // - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path) - // - inputPath is a List[Path] - // - outputFilename is a List[String] // - (key, value) are the tuples that will be saved to the state.yaml file - // - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml) def processedState = config.allArguments .findAll { it.direction == "output" } @@ -1829,7 +1962,7 @@ def publishStatesByConfig(Map args) { // in the state as-is, but is not something that needs // to be copied from the source path to the dest path if (par.type != "file") { - return [[key: plainName_, value: value, inputPath: [], outputFilename: []]] + return [[key: plainName_, value: value]] } // if the orig state does not contain this filename, // it's an optional argument for which the user specified @@ -1860,13 +1993,9 @@ def publishStatesByConfig(Map args) { if (yamlDir != null) { value_ = yamlDir.relativize(value_) } - def inputPath = val instanceof File ? val.toPath() : val - [value: value_, inputPath: inputPath, outputFilename: filename_ix] + return value_ } - def transposedOutputs = ["value", "inputPath", "outputFilename"].collectEntries{ key -> - [key, outputPerFile.collect{dic -> dic[key]}] - } - return [[key: plainName_] + transposedOutputs] + return [["key": plainName_, "value": outputPerFile]] } else { def value_ = java.nio.file.Paths.get(filename) // if id contains a slash @@ -1874,18 +2003,17 @@ def publishStatesByConfig(Map args) { value_ = yamlDir.relativize(value_) } def inputPath = value instanceof File ? value.toPath() : value - return [[key: plainName_, value: value_, inputPath: [inputPath], outputFilename: [filename]]] + return [["key": plainName_, value: value_]] } } + def updatedState_ = processedState.collectEntries{[it.key, it.value]} - def inputPaths = processedState.collectMany{it.inputPath} - def outputFilenames = processedState.collectMany{it.outputFilename} // convert state to yaml blob def yamlBlob_ = toTaggedYamlBlob([id: id_] + updatedState_) - [id_, yamlBlob_, yamlFilename, inputPaths, outputFilenames] + [id_, yamlBlob_, yamlFilename] } | publishStatesProc emit: input_ch @@ -2559,7 +2687,8 @@ def _debug(workflowArgs, debugKey) { def workflowFactory(Map args, Map defaultWfArgs, Map meta) { def workflowArgs = processWorkflowArgs(args, defaultWfArgs, meta) def key_ = workflowArgs["key"] - + def multipleArgs = meta.config.allArguments.findAll{ it.multiple }.collect{it.plainName} + workflow workflowInstance { take: input_ @@ -2716,12 +2845,36 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { } // TODO: move some of the _meta.join_id wrangling to the safeJoin() function. - def chInitialOutput = chArgsWithDefaults + def chInitialOutputMulti = chArgsWithDefaults | _debug(workflowArgs, "processed") // run workflow | innerWorkflowFactory(workflowArgs) - // check output tuple - | map { id_, output_ -> + def chInitialOutputList = chInitialOutputMulti instanceof List ? chInitialOutputMulti : [chInitialOutputMulti] + assert chInitialOutputList.size() > 0: "should have emitted at least one output channel" + // Add a channel ID to the events, which designates the channel the event was emitted from as a running number + // This number is used to sort the events later when the events are gathered from across the channels. + def chInitialOutputListWithIndexedEvents = chInitialOutputList.withIndex().collect{channel, channelIndex -> + def newChannel = channel + | map {tuple -> + assert tuple instanceof List : + "Error in module '${key_}': element in output channel should be a tuple [id, data, ...otherargs...]\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Expected class: List. Found: tuple.getClass() is ${tuple.getClass()}" + + def newEvent = [channelIndex] + tuple + return newEvent + } + return newChannel + } + // Put the events into 1 channel, cover case where there is only one channel is emitted + def chInitialOutput = chInitialOutputList.size() > 1 ? \ + chInitialOutputListWithIndexedEvents[0].mix(*chInitialOutputListWithIndexedEvents.tail()) : \ + chInitialOutputListWithIndexedEvents[0] + def chInitialOutputProcessed = chInitialOutput + | map { tuple -> + def channelId = tuple[0] + def id_ = tuple[1] + def output_ = tuple[2] // see if output map contains metadata def meta_ = @@ -2734,19 +2887,94 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { output_ = output_.findAll{k, v -> k != "_meta"} // check value types - output_ = _processOutputValues(output_, meta.config, id_, key_) + output_ = _checkValidOutputArgument(output_, meta.config, id_, key_) - // simplify output if need be - if (workflowArgs.auto.simplifyOutput && output_.size() == 1) { - output_ = output_.values()[0] - } - - [join_id, id_, output_] + [join_id, channelId, id_, output_] } // | view{"chInitialOutput: ${it.take(3)}"} + // join the output [prev_id, channel_id, new_id, output] with the previous state [prev_id, state, ...] + def chPublishWithPreviousState = safeJoin(chInitialOutputProcessed, chRunFiltered, key_) + // input tuple format: [join_id, channel_id, id, output, prev_state, ...] + // output tuple format: [join_id, channel_id, id, new_state, ...] + | map{ tup -> + def new_state = workflowArgs.toState(tup.drop(2).take(3)) + tup.take(3) + [new_state] + tup.drop(5) + } + if (workflowArgs.auto.publish == "state") { + def chPublishFiles = chPublishWithPreviousState + // input tuple format: [join_id, channel_id, id, new_state, ...] + // output tuple format: [join_id, channel_id, id, new_state] + | map{ tup -> + tup.take(4) + } + + safeJoin(chPublishFiles, chArgsWithDefaults, key_) + // input tuple format: [join_id, channel_id, id, new_state, orig_state, ...] + // output tuple format: [id, new_state, orig_state] + | map { tup -> + tup.drop(2).take(3) + } + | publishFilesByConfig(key: key_, config: meta.config) + } + // Join the state from the events that were emitted from different channels + def chJoined = chInitialOutputProcessed + | map {tuple -> + def join_id = tuple[0] + def channel_id = tuple[1] + def id = tuple[2] + def other = tuple.drop(3) + // Below, groupTuple is used to join the events. To make sure resuming a workflow + // keeps working, the output state must be deterministic. This means the state needs to be + // sorted with groupTuple's has a 'sort' argument. This argument can be set to 'hash', + // but hashing the state when it is large can be problematic in terms of performance. + // Therefore, a custom comparator function is provided. We add the channel ID to the + // states so that we can use the channel ID to sort the items. + def stateWithChannelID = [[channel_id] * other.size(), other].transpose() + // A comparator that is provided to groupTuple's 'sort' argument is applied + // to all elements of the event tuple (that is not the 'id'). The comparator + // closure that is used below expects the input to be List. So the join_id and + // channel_id must also be wrapped in a list. + [[join_id], [channel_id], id] + stateWithChannelID + } + | groupTuple(by: 2, sort: {a, b -> a[0] <=> b[0]}, size: chInitialOutputList.size(), remainder: true) + | map {join_ids, _, id, statesWithChannelID -> + // Remove the channel IDs from the states + def states = statesWithChannelID.collect{it[1]} + def newJoinId = join_ids.flatten().unique{a, b -> a <=> b} + assert newJoinId.size() == 1: "Multiple events were emitted for '$id'." + def newJoinIdUnique = newJoinId[0] + + // Merge the states from the different channels + def newState = states.inject([:]){ old_state, state_to_add -> + return old_state + state_to_add.collectEntries{k, v -> + if (!multipleArgs.contains(k)) { + // if the key is not a multiple argument, we expect only one value + if (old_state.containsKey(k)) { + assert old_state[k] == v : "ID $id: multiple entries for argument $k were emitted." + } + [k, v] + } else { + // if the key is a multiple argument, append the different values into one list + def prevValue = old_state.getOrDefault(k, []) + def prevValueAsList = prevValue instanceof List ? prevValue : [prevValue] + [k, prevValueAsList + v] + } + } + } + + _checkAllRequiredOuputsPresent(newState, meta.config, id, key_) + + // simplify output if need be + if (workflowArgs.auto.simplifyOutput && newState.size() == 1) { + newState = newState.values()[0] + } + + return [newJoinIdUnique, id, newState] + } + // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) + def chNewState = safeJoin(chJoined, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -2755,23 +2983,21 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { } if (workflowArgs.auto.publish == "state") { - def chPublish = chNewState + def chPublishStates = chNewState // input tuple format: [join_id, id, new_state, ...] // output tuple format: [join_id, id, new_state] | map{ tup -> tup.take(3) } - safeJoin(chPublish, chArgsWithDefaults, key_) + safeJoin(chPublishStates, chArgsWithDefaults, key_) // input tuple format: [join_id, id, new_state, orig_state, ...] // output tuple format: [id, new_state, orig_state] | map { tup -> tup.drop(1).take(3) - } + } | publishStatesByConfig(key: key_, config: meta.config) } - - // remove join_id and meta chReturn = chNewState | map { tup -> // input tuple format: [join_id, id, new_state, ...] @@ -2805,7 +3031,7 @@ meta = [ "resources_dir": moduleDir.toRealPath().normalize(), "config": processConfig(readJsonBlob('''{ "name" : "demultiplex", - "version" : "v0.3.8", + "version" : "v0.3.9", "argument_groups" : [ { "name" : "Input arguments", @@ -2957,6 +3183,10 @@ meta = [ } ], "status" : "enabled", + "scope" : { + "image" : "public", + "target" : "public" + }, "requirements" : { "commands" : [ "ps" @@ -3119,14 +3349,14 @@ meta = [ "runner" : "nextflow", "engine" : "native|native", "output" : "target/nextflow/demultiplex", - "viash_version" : "0.9.0", - "git_commit" : "dafe2d829079bd5a9f5eb4e7cf63e92edbf1a742", + "viash_version" : "0.9.4", + "git_commit" : "820318c378d8119e2aa98768282e43c2aa017ba7", "git_remote" : "https://github.com/viash-hub/demultiplex", - "git_tag" : "v0.3.5-9-gdafe2d8" + "git_tag" : "v0.3.8-5-g820318c" }, "package_config" : { "name" : "demultiplex", - "version" : "v0.3.8", + "version" : "v0.3.9", "description" : "Demultiplexing pipeline\n", "info" : { "test_resources" : [ @@ -3136,14 +3366,14 @@ meta = [ } ] }, - "viash_version" : "0.9.0", + "viash_version" : "0.9.4", "source" : "src", "target" : "target", "config_mods" : [ - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n", + ".requirements.commands += ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v0.3.8'" + ".engines[.type == 'docker'].target_tag := 'v0.3.9'" ], "keywords" : [ "bioinformatics", diff --git a/target/nextflow/demultiplex/nextflow.config b/target/nextflow/demultiplex/nextflow.config index 9c91d16..34708f8 100644 --- a/target/nextflow/demultiplex/nextflow.config +++ b/target/nextflow/demultiplex/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'demultiplex' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v0.3.8' + version = 'v0.3.9' description = 'Demultiplexing of raw sequencing data' } diff --git a/target/nextflow/demultiplex/nextflow_schema.json b/target/nextflow/demultiplex/nextflow_schema.json index ec42fc9..9c010c2 100644 --- a/target/nextflow/demultiplex/nextflow_schema.json +++ b/target/nextflow/demultiplex/nextflow_schema.json @@ -69,10 +69,10 @@ "output": { "type": "string", - "description": "Type: `file`, default: `$id.$key.output.output`. Directory to write fastq data to", - "help_text": "Type: `file`, default: `$id.$key.output.output`. Directory to write fastq data to" + "description": "Type: `file`, default: `$id/fastq`. Directory to write fastq data to", + "help_text": "Type: `file`, default: `$id/fastq`. Directory to write fastq data to" , - "default":"$id.$key.output.output" + "default":"$id/fastq" } @@ -80,10 +80,10 @@ "output_falco": { "type": "string", - "description": "Type: List of `file`, default: `$id.$key.output_falco_*.output_falco_*`, multiple_sep: `\";\"`. Directory to write falco output to", - "help_text": "Type: List of `file`, default: `$id.$key.output_falco_*.output_falco_*`, multiple_sep: `\";\"`. Directory to write falco output to" + "description": "Type: List of `file`, default: `$id/qc/fastqc`, multiple_sep: `\";\"`. Directory to write falco output to", + "help_text": "Type: List of `file`, default: `$id/qc/fastqc`, multiple_sep: `\";\"`. Directory to write falco output to" , - "default":"$id.$key.output_falco_*.output_falco_*" + "default":"$id/qc/fastqc" } @@ -91,10 +91,10 @@ "output_multiqc": { "type": "string", - "description": "Type: `file`, default: `$id.$key.output_multiqc.html`. Directory to write falco output to", - "help_text": "Type: `file`, default: `$id.$key.output_multiqc.html`. Directory to write falco output to" + "description": "Type: `file`, default: `$id/qc/multiqc_report.html`. Directory to write falco output to", + "help_text": "Type: `file`, default: `$id/qc/multiqc_report.html`. Directory to write falco output to" , - "default":"$id.$key.output_multiqc.html" + "default":"$id/qc/multiqc_report.html" } @@ -102,10 +102,10 @@ "output_run_information": { "type": "string", - "description": "Type: `file`, required, default: `$id.$key.output_run_information.csv`. ", - "help_text": "Type: `file`, required, default: `$id.$key.output_run_information.csv`. " + "description": "Type: `file`, required, default: `$id/run_information.csv`. ", + "help_text": "Type: `file`, required, default: `$id/run_information.csv`. " , - "default":"$id.$key.output_run_information.csv" + "default":"$id/run_information.csv" } diff --git a/target/nextflow/io/interop_summary_to_csv/.config.vsh.yaml b/target/nextflow/io/interop_summary_to_csv/.config.vsh.yaml index 365fcca..aa393e1 100644 --- a/target/nextflow/io/interop_summary_to_csv/.config.vsh.yaml +++ b/target/nextflow/io/interop_summary_to_csv/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "interop_summary_to_csv" namespace: "io" -version: "v0.3.8" +version: "v0.3.9" argument_groups: - name: "Input arguments" arguments: @@ -43,8 +43,13 @@ resources: dest: "nextflow_labels.config" info: null status: "enabled" +scope: + image: "public" + target: "public" requirements: commands: + - "summary" + - "index-summary" - "ps" license: "MIT" links: @@ -121,7 +126,7 @@ engines: id: "docker" image: "debian:stable-slim" target_registry: "images.viash-hub.com" - target_tag: "v0.3.8" + target_tag: "v0.3.9" namespace_separator: "/" setup: - type: "apt" @@ -145,29 +150,29 @@ build_info: engine: "docker|native" output: "target/nextflow/io/interop_summary_to_csv" executable: "target/nextflow/io/interop_summary_to_csv/main.nf" - viash_version: "0.9.0" - git_commit: "dafe2d829079bd5a9f5eb4e7cf63e92edbf1a742" + viash_version: "0.9.4" + git_commit: "820318c378d8119e2aa98768282e43c2aa017ba7" git_remote: "https://github.com/viash-hub/demultiplex" - git_tag: "v0.3.5-9-gdafe2d8" + git_tag: "v0.3.8-5-g820318c" package_config: name: "demultiplex" - version: "v0.3.8" + version: "v0.3.9" description: "Demultiplexing pipeline\n" info: test_resources: - path: "gs://viash-hub-test-data/demultiplex/v2/" dest: "testData" - viash_version: "0.9.0" + viash_version: "0.9.4" source: "src" target: "target" config_mods: - - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\ + - ".requirements.commands += ['ps']\n.runners[.type == 'nextflow'].directives.tag\ \ := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n\ .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ )'\n" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.3.8'" + - ".engines[.type == 'docker'].target_tag := 'v0.3.9'" keywords: - "bioinformatics" - "sequence" diff --git a/target/nextflow/io/interop_summary_to_csv/main.nf b/target/nextflow/io/interop_summary_to_csv/main.nf index dc8b1a0..986ad85 100644 --- a/target/nextflow/io/interop_summary_to_csv/main.nf +++ b/target/nextflow/io/interop_summary_to_csv/main.nf @@ -1,6 +1,6 @@ -// interop_summary_to_csv v0.3.8 +// interop_summary_to_csv v0.3.9 // -// This wrapper script is auto-generated by viash 0.9.0 and is thus a derivative +// This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data // Intuitive. // @@ -82,64 +82,56 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi foundClass = "List[${e.foundClass}]" } } else if (par.type == "string") { - // cast to string if need be + // cast to string if need be. only cast if the value is a GString if (value instanceof GString) { - value = value.toString() + value = value as String } expectedClass = value instanceof String ? null : "String" } else if (par.type == "integer") { // cast to integer if need be - if (value instanceof String) { + if (value !instanceof Integer) { try { - value = value.toInteger() + value = value as Integer } catch (NumberFormatException e) { - // do nothing + expectedClass = "Integer" } } - if (value instanceof java.math.BigInteger) { - value = value.intValue() - } - expectedClass = value instanceof Integer ? null : "Integer" } else if (par.type == "long") { // cast to long if need be - if (value instanceof String) { + if (value !instanceof Long) { try { - value = value.toLong() + value = value as Long } catch (NumberFormatException e) { - // do nothing + expectedClass = "Long" } } - if (value instanceof Integer) { - value = value.toLong() - } - expectedClass = value instanceof Long ? null : "Long" } else if (par.type == "double") { // cast to double if need be - if (value instanceof String) { + if (value !instanceof Double) { try { - value = value.toDouble() + value = value as Double } catch (NumberFormatException e) { - // do nothing + expectedClass = "Double" } } - if (value instanceof java.math.BigDecimal) { - value = value.doubleValue() + } else if (par.type == "float") { + // cast to float if need be + if (value !instanceof Float) { + try { + value = value as Float + } catch (NumberFormatException e) { + expectedClass = "Float" + } } - if (value instanceof Float) { - value = value.toDouble() - } - expectedClass = value instanceof Double ? null : "Double" } else if (par.type == "boolean" | par.type == "boolean_true" | par.type == "boolean_false") { // cast to boolean if need be - if (value instanceof String) { - def valueLower = value.toLowerCase() - if (valueLower == "true") { - value = true - } else if (valueLower == "false") { - value = false + if (value !instanceof Boolean) { + try { + value = value as Boolean + } catch (Exception e) { + expectedClass = "Boolean" } } - expectedClass = value instanceof Boolean ? null : "Boolean" } else if (par.type == "file" && (par.direction == "input" || stage == "output")) { // cast to path if need be if (value instanceof String) { @@ -151,10 +143,13 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi expectedClass = value instanceof Path ? null : "Path" } else if (par.type == "file" && stage == "input" && par.direction == "output") { // cast to string if need be - if (value instanceof GString) { - value = value.toString() + if (value !instanceof String) { + try { + value = value as String + } catch (Exception e) { + expectedClass = "String" + } } - expectedClass = value instanceof String ? null : "String" } else { // didn't find a match for par.type expectedClass = par.type @@ -173,7 +168,7 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi Map _processInputValues(Map inputs, Map config, String id, String key) { if (!workflow.stubRun) { config.allArguments.each { arg -> - if (arg.required) { + if (arg.required && arg.direction == "input") { assert inputs.containsKey(arg.plainName) && inputs.get(arg.plainName) != null : "Error in module '${key}' id '${id}': required input argument '${arg.plainName}' is missing" } @@ -192,15 +187,8 @@ Map _processInputValues(Map inputs, Map config, String id, String key) { } // helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_processOutputValues.nf' -Map _processOutputValues(Map outputs, Map config, String id, String key) { +Map _checkValidOutputArgument(Map outputs, Map config, String id, String key) { if (!workflow.stubRun) { - config.allArguments.each { arg -> - if (arg.direction == "output" && arg.required) { - assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : - "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing" - } - } - outputs = outputs.collectEntries { name, value -> def par = config.allArguments.find { it.plainName == name && it.direction == "output" } assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid output argument" @@ -213,6 +201,16 @@ Map _processOutputValues(Map outputs, Map config, String id, String key) { return outputs } +void _checkAllRequiredOuputsPresent(Map outputs, Map config, String id, String key) { + if (!workflow.stubRun) { + config.allArguments.each { arg -> + if (arg.direction == "output" && arg.required) { + assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : + "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing" + } + } + } +} // helper file: 'src/main/resources/io/viash/runners/nextflow/channel/IDChecker.nf' class IDChecker { final def items = [] as Set @@ -1666,6 +1664,162 @@ def joinStates(Closure apply_) { } return joinStatesWf } +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishFiles.nf' +def publishFiles(Map args) { + def key_ = args.get("key") + + assert key_ != null : "publishFiles: key must be specified" + + workflow publishFilesWf { + take: input_ch + main: + input_ch + | map { tup -> + def id_ = tup[0] + def state_ = tup[1] + + // the input files and the target output filenames + def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose() + def inputFiles_ = inputoutputFilenames_[0] + def outputFilenames_ = inputoutputFilenames_[1] + + [id_, inputFiles_, outputFilenames_] + } + | publishFilesProc + emit: input_ch + } + return publishFilesWf +} + +process publishFilesProc { + // todo: check publishpath? + publishDir path: "${getPublishDir()}/", mode: "copy" + tag "$id" + input: + tuple val(id), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles) + output: + tuple val(id), path{outputFiles} + script: + def copyCommands = [ + inputFiles instanceof List ? inputFiles : [inputFiles], + outputFiles instanceof List ? outputFiles : [outputFiles] + ] + .transpose() + .collectMany{infile, outfile -> + if (infile.toString() != outfile.toString()) { + [ + "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"", + "cp -r '${infile.toString()}' '${outfile.toString()}'" + ] + } else { + // no need to copy if infile is the same as outfile + [] + } + } + """ + echo "Copying output files to destination folder" + ${copyCommands.join("\n ")} + """ +} + + +// this assumes that the state contains no other values other than those specified in the config +def publishFilesByConfig(Map args) { + def config = args.get("config") + assert config != null : "publishFilesByConfig: config must be specified" + + def key_ = args.get("key", config.name) + assert key_ != null : "publishFilesByConfig: key must be specified" + + workflow publishFilesSimpleWf { + take: input_ch + main: + input_ch + | map { tup -> + def id_ = tup[0] + def state_ = tup[1] // e.g. [output: new File("myoutput.h5ad"), k: 10] + def origState_ = tup[2] // e.g. [output: '$id.$key.foo.h5ad'] + + + // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where + // - key is a String + // - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path) + // - inputPath is a List[Path] + // - outputFilename is a List[String] + // - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml) + def processedState = + config.allArguments + .findAll { it.direction == "output" } + .collectMany { par -> + def plainName_ = par.plainName + // if the state does not contain the key, it's an + // optional argument for which the component did + // not generate any output OR multiple channels were emitted + // and the output was just not added to using the channel + // that is now being parsed + if (!state_.containsKey(plainName_)) { + return [] + } + def value = state_[plainName_] + // if the parameter is not a file, it should be stored + // in the state as-is, but is not something that needs + // to be copied from the source path to the dest path + if (par.type != "file") { + return [[inputPath: [], outputFilename: []]] + } + // if the orig state does not contain this filename, + // it's an optional argument for which the user specified + // that it should not be returned as a state + if (!origState_.containsKey(plainName_)) { + return [] + } + def filenameTemplate = origState_[plainName_] + // if the pararameter is multiple: true, fetch the template + if (par.multiple && filenameTemplate instanceof List) { + filenameTemplate = filenameTemplate[0] + } + // instantiate the template + def filename = filenameTemplate + .replaceAll('\\$id', id_) + .replaceAll('\\$\\{id\\}', id_) + .replaceAll('\\$key', key_) + .replaceAll('\\$\\{key\\}', key_) + if (par.multiple) { + // if the parameter is multiple: true, the filename + // should contain a wildcard '*' that is replaced with + // the index of the file + assert filename.contains("*") : "Module '${key_}' id '${id_}': Multiple output files specified, but no wildcard '*' in the filename: ${filename}" + def outputPerFile = value.withIndex().collect{ val, ix -> + def filename_ix = filename.replace("*", ix.toString()) + def inputPath = val instanceof File ? val.toPath() : val + [inputPath: inputPath, outputFilename: filename_ix] + } + def transposedOutputs = ["inputPath", "outputFilename"].collectEntries{ key -> + [key, outputPerFile.collect{dic -> dic[key]}] + } + return [[key: plainName_] + transposedOutputs] + } else { + def value_ = java.nio.file.Paths.get(filename) + def inputPath = value instanceof File ? value.toPath() : value + return [[inputPath: [inputPath], outputFilename: [filename]]] + } + } + + def inputPaths = processedState.collectMany{it.inputPath} + def outputFilenames = processedState.collectMany{it.outputFilename} + + + [id_, inputPaths, outputFilenames] + } + | publishFilesProc + emit: input_ch + } + return publishFilesSimpleWf +} + + + + // helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishStates.nf' def collectFiles(obj) { if (obj instanceof java.io.File || obj instanceof Path) { @@ -1723,8 +1877,6 @@ def publishStates(Map args) { // the input files and the target output filenames def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose() - def inputFiles_ = inputoutputFilenames_[0] - def outputFilenames_ = inputoutputFilenames_[1] def yamlFilename = yamlTemplate_ .replaceAll('\\$id', id_) @@ -1737,7 +1889,7 @@ def publishStates(Map args) { // convert state to yaml blob def yamlBlob_ = toRelativeTaggedYamlBlob([id: id_] + state_, java.nio.file.Paths.get(yamlFilename)) - [id_, yamlBlob_, yamlFilename, inputFiles_, outputFilenames_] + [id_, yamlBlob_, yamlFilename] } | publishStatesProc emit: input_ch @@ -1749,33 +1901,17 @@ process publishStatesProc { publishDir path: "${getPublishDir()}/", mode: "copy" tag "$id" input: - tuple val(id), val(yamlBlob), val(yamlFile), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles) + tuple val(id), val(yamlBlob), val(yamlFile) output: - tuple val(id), path{[yamlFile] + outputFiles} + tuple val(id), path{[yamlFile]} script: - def copyCommands = [ - inputFiles instanceof List ? inputFiles : [inputFiles], - outputFiles instanceof List ? outputFiles : [outputFiles] - ] - .transpose() - .collectMany{infile, outfile -> - if (infile.toString() != outfile.toString()) { - [ - "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"", - "cp -r '${infile.toString()}' '${outfile.toString()}'" - ] - } else { - // no need to copy if infile is the same as outfile - [] - } - } """ -mkdir -p "\$(dirname '${yamlFile}')" -echo "Storing state as yaml" -echo '${yamlBlob}' > '${yamlFile}' -echo "Copying output files to destination folder" -${copyCommands.join("\n ")} -""" + mkdir -p "\$(dirname '${yamlFile}')" + echo "Storing state as yaml" + cat > '${yamlFile}' << HERE +${yamlBlob} +HERE + """ } @@ -1806,13 +1942,10 @@ def publishStatesByConfig(Map args) { .replaceAll('\\$\\{key\\}', key_) def yamlDir = java.nio.file.Paths.get(yamlFilename).getParent() - // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where + // the processed state is a list of [key, value] tuples, where // - key is a String // - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path) - // - inputPath is a List[Path] - // - outputFilename is a List[String] // - (key, value) are the tuples that will be saved to the state.yaml file - // - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml) def processedState = config.allArguments .findAll { it.direction == "output" } @@ -1829,7 +1962,7 @@ def publishStatesByConfig(Map args) { // in the state as-is, but is not something that needs // to be copied from the source path to the dest path if (par.type != "file") { - return [[key: plainName_, value: value, inputPath: [], outputFilename: []]] + return [[key: plainName_, value: value]] } // if the orig state does not contain this filename, // it's an optional argument for which the user specified @@ -1860,13 +1993,9 @@ def publishStatesByConfig(Map args) { if (yamlDir != null) { value_ = yamlDir.relativize(value_) } - def inputPath = val instanceof File ? val.toPath() : val - [value: value_, inputPath: inputPath, outputFilename: filename_ix] + return value_ } - def transposedOutputs = ["value", "inputPath", "outputFilename"].collectEntries{ key -> - [key, outputPerFile.collect{dic -> dic[key]}] - } - return [[key: plainName_] + transposedOutputs] + return [["key": plainName_, "value": outputPerFile]] } else { def value_ = java.nio.file.Paths.get(filename) // if id contains a slash @@ -1874,18 +2003,17 @@ def publishStatesByConfig(Map args) { value_ = yamlDir.relativize(value_) } def inputPath = value instanceof File ? value.toPath() : value - return [[key: plainName_, value: value_, inputPath: [inputPath], outputFilename: [filename]]] + return [["key": plainName_, value: value_]] } } + def updatedState_ = processedState.collectEntries{[it.key, it.value]} - def inputPaths = processedState.collectMany{it.inputPath} - def outputFilenames = processedState.collectMany{it.outputFilename} // convert state to yaml blob def yamlBlob_ = toTaggedYamlBlob([id: id_] + updatedState_) - [id_, yamlBlob_, yamlFilename, inputPaths, outputFilenames] + [id_, yamlBlob_, yamlFilename] } | publishStatesProc emit: input_ch @@ -2559,7 +2687,8 @@ def _debug(workflowArgs, debugKey) { def workflowFactory(Map args, Map defaultWfArgs, Map meta) { def workflowArgs = processWorkflowArgs(args, defaultWfArgs, meta) def key_ = workflowArgs["key"] - + def multipleArgs = meta.config.allArguments.findAll{ it.multiple }.collect{it.plainName} + workflow workflowInstance { take: input_ @@ -2716,12 +2845,36 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { } // TODO: move some of the _meta.join_id wrangling to the safeJoin() function. - def chInitialOutput = chArgsWithDefaults + def chInitialOutputMulti = chArgsWithDefaults | _debug(workflowArgs, "processed") // run workflow | innerWorkflowFactory(workflowArgs) - // check output tuple - | map { id_, output_ -> + def chInitialOutputList = chInitialOutputMulti instanceof List ? chInitialOutputMulti : [chInitialOutputMulti] + assert chInitialOutputList.size() > 0: "should have emitted at least one output channel" + // Add a channel ID to the events, which designates the channel the event was emitted from as a running number + // This number is used to sort the events later when the events are gathered from across the channels. + def chInitialOutputListWithIndexedEvents = chInitialOutputList.withIndex().collect{channel, channelIndex -> + def newChannel = channel + | map {tuple -> + assert tuple instanceof List : + "Error in module '${key_}': element in output channel should be a tuple [id, data, ...otherargs...]\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Expected class: List. Found: tuple.getClass() is ${tuple.getClass()}" + + def newEvent = [channelIndex] + tuple + return newEvent + } + return newChannel + } + // Put the events into 1 channel, cover case where there is only one channel is emitted + def chInitialOutput = chInitialOutputList.size() > 1 ? \ + chInitialOutputListWithIndexedEvents[0].mix(*chInitialOutputListWithIndexedEvents.tail()) : \ + chInitialOutputListWithIndexedEvents[0] + def chInitialOutputProcessed = chInitialOutput + | map { tuple -> + def channelId = tuple[0] + def id_ = tuple[1] + def output_ = tuple[2] // see if output map contains metadata def meta_ = @@ -2734,19 +2887,94 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { output_ = output_.findAll{k, v -> k != "_meta"} // check value types - output_ = _processOutputValues(output_, meta.config, id_, key_) + output_ = _checkValidOutputArgument(output_, meta.config, id_, key_) - // simplify output if need be - if (workflowArgs.auto.simplifyOutput && output_.size() == 1) { - output_ = output_.values()[0] - } - - [join_id, id_, output_] + [join_id, channelId, id_, output_] } // | view{"chInitialOutput: ${it.take(3)}"} + // join the output [prev_id, channel_id, new_id, output] with the previous state [prev_id, state, ...] + def chPublishWithPreviousState = safeJoin(chInitialOutputProcessed, chRunFiltered, key_) + // input tuple format: [join_id, channel_id, id, output, prev_state, ...] + // output tuple format: [join_id, channel_id, id, new_state, ...] + | map{ tup -> + def new_state = workflowArgs.toState(tup.drop(2).take(3)) + tup.take(3) + [new_state] + tup.drop(5) + } + if (workflowArgs.auto.publish == "state") { + def chPublishFiles = chPublishWithPreviousState + // input tuple format: [join_id, channel_id, id, new_state, ...] + // output tuple format: [join_id, channel_id, id, new_state] + | map{ tup -> + tup.take(4) + } + + safeJoin(chPublishFiles, chArgsWithDefaults, key_) + // input tuple format: [join_id, channel_id, id, new_state, orig_state, ...] + // output tuple format: [id, new_state, orig_state] + | map { tup -> + tup.drop(2).take(3) + } + | publishFilesByConfig(key: key_, config: meta.config) + } + // Join the state from the events that were emitted from different channels + def chJoined = chInitialOutputProcessed + | map {tuple -> + def join_id = tuple[0] + def channel_id = tuple[1] + def id = tuple[2] + def other = tuple.drop(3) + // Below, groupTuple is used to join the events. To make sure resuming a workflow + // keeps working, the output state must be deterministic. This means the state needs to be + // sorted with groupTuple's has a 'sort' argument. This argument can be set to 'hash', + // but hashing the state when it is large can be problematic in terms of performance. + // Therefore, a custom comparator function is provided. We add the channel ID to the + // states so that we can use the channel ID to sort the items. + def stateWithChannelID = [[channel_id] * other.size(), other].transpose() + // A comparator that is provided to groupTuple's 'sort' argument is applied + // to all elements of the event tuple (that is not the 'id'). The comparator + // closure that is used below expects the input to be List. So the join_id and + // channel_id must also be wrapped in a list. + [[join_id], [channel_id], id] + stateWithChannelID + } + | groupTuple(by: 2, sort: {a, b -> a[0] <=> b[0]}, size: chInitialOutputList.size(), remainder: true) + | map {join_ids, _, id, statesWithChannelID -> + // Remove the channel IDs from the states + def states = statesWithChannelID.collect{it[1]} + def newJoinId = join_ids.flatten().unique{a, b -> a <=> b} + assert newJoinId.size() == 1: "Multiple events were emitted for '$id'." + def newJoinIdUnique = newJoinId[0] + + // Merge the states from the different channels + def newState = states.inject([:]){ old_state, state_to_add -> + return old_state + state_to_add.collectEntries{k, v -> + if (!multipleArgs.contains(k)) { + // if the key is not a multiple argument, we expect only one value + if (old_state.containsKey(k)) { + assert old_state[k] == v : "ID $id: multiple entries for argument $k were emitted." + } + [k, v] + } else { + // if the key is a multiple argument, append the different values into one list + def prevValue = old_state.getOrDefault(k, []) + def prevValueAsList = prevValue instanceof List ? prevValue : [prevValue] + [k, prevValueAsList + v] + } + } + } + + _checkAllRequiredOuputsPresent(newState, meta.config, id, key_) + + // simplify output if need be + if (workflowArgs.auto.simplifyOutput && newState.size() == 1) { + newState = newState.values()[0] + } + + return [newJoinIdUnique, id, newState] + } + // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) + def chNewState = safeJoin(chJoined, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -2755,23 +2983,21 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { } if (workflowArgs.auto.publish == "state") { - def chPublish = chNewState + def chPublishStates = chNewState // input tuple format: [join_id, id, new_state, ...] // output tuple format: [join_id, id, new_state] | map{ tup -> tup.take(3) } - safeJoin(chPublish, chArgsWithDefaults, key_) + safeJoin(chPublishStates, chArgsWithDefaults, key_) // input tuple format: [join_id, id, new_state, orig_state, ...] // output tuple format: [id, new_state, orig_state] | map { tup -> tup.drop(1).take(3) - } + } | publishStatesByConfig(key: key_, config: meta.config) } - - // remove join_id and meta chReturn = chNewState | map { tup -> // input tuple format: [join_id, id, new_state, ...] @@ -2806,7 +3032,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "interop_summary_to_csv", "namespace" : "io", - "version" : "v0.3.8", + "version" : "v0.3.9", "argument_groups" : [ { "name" : "Input arguments", @@ -2863,8 +3089,14 @@ meta = [ } ], "status" : "enabled", + "scope" : { + "image" : "public", + "target" : "public" + }, "requirements" : { "commands" : [ + "summary", + "index-summary", "ps" ] }, @@ -2955,7 +3187,7 @@ meta = [ "id" : "docker", "image" : "debian:stable-slim", "target_registry" : "images.viash-hub.com", - "target_tag" : "v0.3.8", + "target_tag" : "v0.3.9", "namespace_separator" : "/", "setup" : [ { @@ -2984,14 +3216,14 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/io/interop_summary_to_csv", - "viash_version" : "0.9.0", - "git_commit" : "dafe2d829079bd5a9f5eb4e7cf63e92edbf1a742", + "viash_version" : "0.9.4", + "git_commit" : "820318c378d8119e2aa98768282e43c2aa017ba7", "git_remote" : "https://github.com/viash-hub/demultiplex", - "git_tag" : "v0.3.5-9-gdafe2d8" + "git_tag" : "v0.3.8-5-g820318c" }, "package_config" : { "name" : "demultiplex", - "version" : "v0.3.8", + "version" : "v0.3.9", "description" : "Demultiplexing pipeline\n", "info" : { "test_resources" : [ @@ -3001,14 +3233,14 @@ meta = [ } ] }, - "viash_version" : "0.9.0", + "viash_version" : "0.9.4", "source" : "src", "target" : "target", "config_mods" : [ - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n", + ".requirements.commands += ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v0.3.8'" + ".engines[.type == 'docker'].target_tag := 'v0.3.9'" ], "keywords" : [ "bioinformatics", @@ -3404,7 +3636,7 @@ def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) { // create process from temp file def binding = new nextflow.script.ScriptBinding([:]) def session = nextflow.Nextflow.getSession() - def parser = new nextflow.script.ScriptParser(session) + def parser = _getScriptLoader(session) .setModule(true) .setBinding(binding) def moduleScript = parser.runScript(tempFile) @@ -3418,6 +3650,27 @@ def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) { return scriptMeta.getProcess(procKey) } +// use Reflection to get a ScriptParser / ScriptLoader +// <25.02.0-edge: new nextflow.script.ScriptParser(session) +// >=25.02.0-edge: nextflow.script.ScriptLoaderFactory.create(session) +def _getScriptLoader(nextflow.Session session) { + // try using the old method + try { + Class scriptParserClass = Class.forName('nextflow.script.ScriptParser') + return scriptParserClass.getDeclaredConstructor(nextflow.Session).newInstance(session) + } catch (ClassNotFoundException e) { + // else try with the new method + try { + Class scriptLoaderFactoryClass = Class.forName('nextflow.script.ScriptLoaderFactory') + def createMethod = scriptLoaderFactoryClass.getDeclaredMethod('create', nextflow.Session) + return createMethod.invoke(null, session) // null because create is static + } catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException | java.lang.reflect.InvocationTargetException e2) { + // Handle the case where neither class is found + throw new Exception("Neither nextflow.script.ScriptParser nor nextflow.script.ScriptLoaderFactory could be found. Is this a compatible Nextflow version?", e2) + } + } +} + // defaults meta["defaults"] = [ // key to be used to trace the process and determine output names @@ -3431,7 +3684,7 @@ meta["defaults"] = [ "container" : { "registry" : "images.viash-hub.com", "image" : "vsh/demultiplex/io/interop_summary_to_csv", - "tag" : "v0.3.8" + "tag" : "v0.3.9" }, "tag" : "$id" }'''), diff --git a/target/nextflow/io/interop_summary_to_csv/nextflow.config b/target/nextflow/io/interop_summary_to_csv/nextflow.config index 8b52e74..fbaae9e 100644 --- a/target/nextflow/io/interop_summary_to_csv/nextflow.config +++ b/target/nextflow/io/interop_summary_to_csv/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'io/interop_summary_to_csv' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v0.3.8' + version = 'v0.3.9' } process.container = 'nextflow/bash:latest' diff --git a/target/nextflow/io/interop_summary_to_csv/nextflow_schema.json b/target/nextflow/io/interop_summary_to_csv/nextflow_schema.json index ca1fafd..2b62b98 100644 --- a/target/nextflow/io/interop_summary_to_csv/nextflow_schema.json +++ b/target/nextflow/io/interop_summary_to_csv/nextflow_schema.json @@ -37,10 +37,10 @@ "output_run_summary": { "type": "string", - "description": "Type: `file`, required, default: `$id.$key.output_run_summary.output_run_summary`. ", - "help_text": "Type: `file`, required, default: `$id.$key.output_run_summary.output_run_summary`. " + "description": "Type: `file`, required, default: `$id.$key.output_run_summary`. ", + "help_text": "Type: `file`, required, default: `$id.$key.output_run_summary`. " , - "default":"$id.$key.output_run_summary.output_run_summary" + "default":"$id.$key.output_run_summary" } @@ -48,10 +48,10 @@ "output_index_summary": { "type": "string", - "description": "Type: `file`, required, default: `$id.$key.output_index_summary.output_index_summary`. ", - "help_text": "Type: `file`, required, default: `$id.$key.output_index_summary.output_index_summary`. " + "description": "Type: `file`, required, default: `$id.$key.output_index_summary`. ", + "help_text": "Type: `file`, required, default: `$id.$key.output_index_summary`. " , - "default":"$id.$key.output_index_summary.output_index_summary" + "default":"$id.$key.output_index_summary" } diff --git a/target/nextflow/io/publish/.config.vsh.yaml b/target/nextflow/io/publish/.config.vsh.yaml index b9307ca..2b6b7f0 100644 --- a/target/nextflow/io/publish/.config.vsh.yaml +++ b/target/nextflow/io/publish/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "publish" namespace: "io" -version: "v0.3.8" +version: "v0.3.9" argument_groups: - name: "Input arguments" arguments: @@ -100,6 +100,9 @@ resources: description: "Publish the processed results of the run" info: null status: "enabled" +scope: + image: "public" + target: "public" requirements: commands: - "ps" @@ -178,7 +181,7 @@ engines: id: "docker" image: "debian:stable-slim" target_registry: "images.viash-hub.com" - target_tag: "v0.3.8" + target_tag: "v0.3.9" namespace_separator: "/" setup: - type: "apt" @@ -195,29 +198,29 @@ build_info: engine: "docker|native" output: "target/nextflow/io/publish" executable: "target/nextflow/io/publish/main.nf" - viash_version: "0.9.0" - git_commit: "dafe2d829079bd5a9f5eb4e7cf63e92edbf1a742" + viash_version: "0.9.4" + git_commit: "820318c378d8119e2aa98768282e43c2aa017ba7" git_remote: "https://github.com/viash-hub/demultiplex" - git_tag: "v0.3.5-9-gdafe2d8" + git_tag: "v0.3.8-5-g820318c" package_config: name: "demultiplex" - version: "v0.3.8" + version: "v0.3.9" description: "Demultiplexing pipeline\n" info: test_resources: - path: "gs://viash-hub-test-data/demultiplex/v2/" dest: "testData" - viash_version: "0.9.0" + viash_version: "0.9.4" source: "src" target: "target" config_mods: - - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\ + - ".requirements.commands += ['ps']\n.runners[.type == 'nextflow'].directives.tag\ \ := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n\ .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ )'\n" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.3.8'" + - ".engines[.type == 'docker'].target_tag := 'v0.3.9'" keywords: - "bioinformatics" - "sequence" diff --git a/target/nextflow/io/publish/main.nf b/target/nextflow/io/publish/main.nf index 5c26357..66e9239 100644 --- a/target/nextflow/io/publish/main.nf +++ b/target/nextflow/io/publish/main.nf @@ -1,6 +1,6 @@ -// publish v0.3.8 +// publish v0.3.9 // -// This wrapper script is auto-generated by viash 0.9.0 and is thus a derivative +// This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data // Intuitive. // @@ -82,64 +82,56 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi foundClass = "List[${e.foundClass}]" } } else if (par.type == "string") { - // cast to string if need be + // cast to string if need be. only cast if the value is a GString if (value instanceof GString) { - value = value.toString() + value = value as String } expectedClass = value instanceof String ? null : "String" } else if (par.type == "integer") { // cast to integer if need be - if (value instanceof String) { + if (value !instanceof Integer) { try { - value = value.toInteger() + value = value as Integer } catch (NumberFormatException e) { - // do nothing + expectedClass = "Integer" } } - if (value instanceof java.math.BigInteger) { - value = value.intValue() - } - expectedClass = value instanceof Integer ? null : "Integer" } else if (par.type == "long") { // cast to long if need be - if (value instanceof String) { + if (value !instanceof Long) { try { - value = value.toLong() + value = value as Long } catch (NumberFormatException e) { - // do nothing + expectedClass = "Long" } } - if (value instanceof Integer) { - value = value.toLong() - } - expectedClass = value instanceof Long ? null : "Long" } else if (par.type == "double") { // cast to double if need be - if (value instanceof String) { + if (value !instanceof Double) { try { - value = value.toDouble() + value = value as Double } catch (NumberFormatException e) { - // do nothing + expectedClass = "Double" } } - if (value instanceof java.math.BigDecimal) { - value = value.doubleValue() + } else if (par.type == "float") { + // cast to float if need be + if (value !instanceof Float) { + try { + value = value as Float + } catch (NumberFormatException e) { + expectedClass = "Float" + } } - if (value instanceof Float) { - value = value.toDouble() - } - expectedClass = value instanceof Double ? null : "Double" } else if (par.type == "boolean" | par.type == "boolean_true" | par.type == "boolean_false") { // cast to boolean if need be - if (value instanceof String) { - def valueLower = value.toLowerCase() - if (valueLower == "true") { - value = true - } else if (valueLower == "false") { - value = false + if (value !instanceof Boolean) { + try { + value = value as Boolean + } catch (Exception e) { + expectedClass = "Boolean" } } - expectedClass = value instanceof Boolean ? null : "Boolean" } else if (par.type == "file" && (par.direction == "input" || stage == "output")) { // cast to path if need be if (value instanceof String) { @@ -151,10 +143,13 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi expectedClass = value instanceof Path ? null : "Path" } else if (par.type == "file" && stage == "input" && par.direction == "output") { // cast to string if need be - if (value instanceof GString) { - value = value.toString() + if (value !instanceof String) { + try { + value = value as String + } catch (Exception e) { + expectedClass = "String" + } } - expectedClass = value instanceof String ? null : "String" } else { // didn't find a match for par.type expectedClass = par.type @@ -173,7 +168,7 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi Map _processInputValues(Map inputs, Map config, String id, String key) { if (!workflow.stubRun) { config.allArguments.each { arg -> - if (arg.required) { + if (arg.required && arg.direction == "input") { assert inputs.containsKey(arg.plainName) && inputs.get(arg.plainName) != null : "Error in module '${key}' id '${id}': required input argument '${arg.plainName}' is missing" } @@ -192,15 +187,8 @@ Map _processInputValues(Map inputs, Map config, String id, String key) { } // helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_processOutputValues.nf' -Map _processOutputValues(Map outputs, Map config, String id, String key) { +Map _checkValidOutputArgument(Map outputs, Map config, String id, String key) { if (!workflow.stubRun) { - config.allArguments.each { arg -> - if (arg.direction == "output" && arg.required) { - assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : - "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing" - } - } - outputs = outputs.collectEntries { name, value -> def par = config.allArguments.find { it.plainName == name && it.direction == "output" } assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid output argument" @@ -213,6 +201,16 @@ Map _processOutputValues(Map outputs, Map config, String id, String key) { return outputs } +void _checkAllRequiredOuputsPresent(Map outputs, Map config, String id, String key) { + if (!workflow.stubRun) { + config.allArguments.each { arg -> + if (arg.direction == "output" && arg.required) { + assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : + "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing" + } + } + } +} // helper file: 'src/main/resources/io/viash/runners/nextflow/channel/IDChecker.nf' class IDChecker { final def items = [] as Set @@ -1666,6 +1664,162 @@ def joinStates(Closure apply_) { } return joinStatesWf } +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishFiles.nf' +def publishFiles(Map args) { + def key_ = args.get("key") + + assert key_ != null : "publishFiles: key must be specified" + + workflow publishFilesWf { + take: input_ch + main: + input_ch + | map { tup -> + def id_ = tup[0] + def state_ = tup[1] + + // the input files and the target output filenames + def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose() + def inputFiles_ = inputoutputFilenames_[0] + def outputFilenames_ = inputoutputFilenames_[1] + + [id_, inputFiles_, outputFilenames_] + } + | publishFilesProc + emit: input_ch + } + return publishFilesWf +} + +process publishFilesProc { + // todo: check publishpath? + publishDir path: "${getPublishDir()}/", mode: "copy" + tag "$id" + input: + tuple val(id), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles) + output: + tuple val(id), path{outputFiles} + script: + def copyCommands = [ + inputFiles instanceof List ? inputFiles : [inputFiles], + outputFiles instanceof List ? outputFiles : [outputFiles] + ] + .transpose() + .collectMany{infile, outfile -> + if (infile.toString() != outfile.toString()) { + [ + "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"", + "cp -r '${infile.toString()}' '${outfile.toString()}'" + ] + } else { + // no need to copy if infile is the same as outfile + [] + } + } + """ + echo "Copying output files to destination folder" + ${copyCommands.join("\n ")} + """ +} + + +// this assumes that the state contains no other values other than those specified in the config +def publishFilesByConfig(Map args) { + def config = args.get("config") + assert config != null : "publishFilesByConfig: config must be specified" + + def key_ = args.get("key", config.name) + assert key_ != null : "publishFilesByConfig: key must be specified" + + workflow publishFilesSimpleWf { + take: input_ch + main: + input_ch + | map { tup -> + def id_ = tup[0] + def state_ = tup[1] // e.g. [output: new File("myoutput.h5ad"), k: 10] + def origState_ = tup[2] // e.g. [output: '$id.$key.foo.h5ad'] + + + // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where + // - key is a String + // - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path) + // - inputPath is a List[Path] + // - outputFilename is a List[String] + // - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml) + def processedState = + config.allArguments + .findAll { it.direction == "output" } + .collectMany { par -> + def plainName_ = par.plainName + // if the state does not contain the key, it's an + // optional argument for which the component did + // not generate any output OR multiple channels were emitted + // and the output was just not added to using the channel + // that is now being parsed + if (!state_.containsKey(plainName_)) { + return [] + } + def value = state_[plainName_] + // if the parameter is not a file, it should be stored + // in the state as-is, but is not something that needs + // to be copied from the source path to the dest path + if (par.type != "file") { + return [[inputPath: [], outputFilename: []]] + } + // if the orig state does not contain this filename, + // it's an optional argument for which the user specified + // that it should not be returned as a state + if (!origState_.containsKey(plainName_)) { + return [] + } + def filenameTemplate = origState_[plainName_] + // if the pararameter is multiple: true, fetch the template + if (par.multiple && filenameTemplate instanceof List) { + filenameTemplate = filenameTemplate[0] + } + // instantiate the template + def filename = filenameTemplate + .replaceAll('\\$id', id_) + .replaceAll('\\$\\{id\\}', id_) + .replaceAll('\\$key', key_) + .replaceAll('\\$\\{key\\}', key_) + if (par.multiple) { + // if the parameter is multiple: true, the filename + // should contain a wildcard '*' that is replaced with + // the index of the file + assert filename.contains("*") : "Module '${key_}' id '${id_}': Multiple output files specified, but no wildcard '*' in the filename: ${filename}" + def outputPerFile = value.withIndex().collect{ val, ix -> + def filename_ix = filename.replace("*", ix.toString()) + def inputPath = val instanceof File ? val.toPath() : val + [inputPath: inputPath, outputFilename: filename_ix] + } + def transposedOutputs = ["inputPath", "outputFilename"].collectEntries{ key -> + [key, outputPerFile.collect{dic -> dic[key]}] + } + return [[key: plainName_] + transposedOutputs] + } else { + def value_ = java.nio.file.Paths.get(filename) + def inputPath = value instanceof File ? value.toPath() : value + return [[inputPath: [inputPath], outputFilename: [filename]]] + } + } + + def inputPaths = processedState.collectMany{it.inputPath} + def outputFilenames = processedState.collectMany{it.outputFilename} + + + [id_, inputPaths, outputFilenames] + } + | publishFilesProc + emit: input_ch + } + return publishFilesSimpleWf +} + + + + // helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishStates.nf' def collectFiles(obj) { if (obj instanceof java.io.File || obj instanceof Path) { @@ -1723,8 +1877,6 @@ def publishStates(Map args) { // the input files and the target output filenames def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose() - def inputFiles_ = inputoutputFilenames_[0] - def outputFilenames_ = inputoutputFilenames_[1] def yamlFilename = yamlTemplate_ .replaceAll('\\$id', id_) @@ -1737,7 +1889,7 @@ def publishStates(Map args) { // convert state to yaml blob def yamlBlob_ = toRelativeTaggedYamlBlob([id: id_] + state_, java.nio.file.Paths.get(yamlFilename)) - [id_, yamlBlob_, yamlFilename, inputFiles_, outputFilenames_] + [id_, yamlBlob_, yamlFilename] } | publishStatesProc emit: input_ch @@ -1749,33 +1901,17 @@ process publishStatesProc { publishDir path: "${getPublishDir()}/", mode: "copy" tag "$id" input: - tuple val(id), val(yamlBlob), val(yamlFile), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles) + tuple val(id), val(yamlBlob), val(yamlFile) output: - tuple val(id), path{[yamlFile] + outputFiles} + tuple val(id), path{[yamlFile]} script: - def copyCommands = [ - inputFiles instanceof List ? inputFiles : [inputFiles], - outputFiles instanceof List ? outputFiles : [outputFiles] - ] - .transpose() - .collectMany{infile, outfile -> - if (infile.toString() != outfile.toString()) { - [ - "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"", - "cp -r '${infile.toString()}' '${outfile.toString()}'" - ] - } else { - // no need to copy if infile is the same as outfile - [] - } - } """ -mkdir -p "\$(dirname '${yamlFile}')" -echo "Storing state as yaml" -echo '${yamlBlob}' > '${yamlFile}' -echo "Copying output files to destination folder" -${copyCommands.join("\n ")} -""" + mkdir -p "\$(dirname '${yamlFile}')" + echo "Storing state as yaml" + cat > '${yamlFile}' << HERE +${yamlBlob} +HERE + """ } @@ -1806,13 +1942,10 @@ def publishStatesByConfig(Map args) { .replaceAll('\\$\\{key\\}', key_) def yamlDir = java.nio.file.Paths.get(yamlFilename).getParent() - // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where + // the processed state is a list of [key, value] tuples, where // - key is a String // - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path) - // - inputPath is a List[Path] - // - outputFilename is a List[String] // - (key, value) are the tuples that will be saved to the state.yaml file - // - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml) def processedState = config.allArguments .findAll { it.direction == "output" } @@ -1829,7 +1962,7 @@ def publishStatesByConfig(Map args) { // in the state as-is, but is not something that needs // to be copied from the source path to the dest path if (par.type != "file") { - return [[key: plainName_, value: value, inputPath: [], outputFilename: []]] + return [[key: plainName_, value: value]] } // if the orig state does not contain this filename, // it's an optional argument for which the user specified @@ -1860,13 +1993,9 @@ def publishStatesByConfig(Map args) { if (yamlDir != null) { value_ = yamlDir.relativize(value_) } - def inputPath = val instanceof File ? val.toPath() : val - [value: value_, inputPath: inputPath, outputFilename: filename_ix] + return value_ } - def transposedOutputs = ["value", "inputPath", "outputFilename"].collectEntries{ key -> - [key, outputPerFile.collect{dic -> dic[key]}] - } - return [[key: plainName_] + transposedOutputs] + return [["key": plainName_, "value": outputPerFile]] } else { def value_ = java.nio.file.Paths.get(filename) // if id contains a slash @@ -1874,18 +2003,17 @@ def publishStatesByConfig(Map args) { value_ = yamlDir.relativize(value_) } def inputPath = value instanceof File ? value.toPath() : value - return [[key: plainName_, value: value_, inputPath: [inputPath], outputFilename: [filename]]] + return [["key": plainName_, value: value_]] } } + def updatedState_ = processedState.collectEntries{[it.key, it.value]} - def inputPaths = processedState.collectMany{it.inputPath} - def outputFilenames = processedState.collectMany{it.outputFilename} // convert state to yaml blob def yamlBlob_ = toTaggedYamlBlob([id: id_] + updatedState_) - [id_, yamlBlob_, yamlFilename, inputPaths, outputFilenames] + [id_, yamlBlob_, yamlFilename] } | publishStatesProc emit: input_ch @@ -2559,7 +2687,8 @@ def _debug(workflowArgs, debugKey) { def workflowFactory(Map args, Map defaultWfArgs, Map meta) { def workflowArgs = processWorkflowArgs(args, defaultWfArgs, meta) def key_ = workflowArgs["key"] - + def multipleArgs = meta.config.allArguments.findAll{ it.multiple }.collect{it.plainName} + workflow workflowInstance { take: input_ @@ -2716,12 +2845,36 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { } // TODO: move some of the _meta.join_id wrangling to the safeJoin() function. - def chInitialOutput = chArgsWithDefaults + def chInitialOutputMulti = chArgsWithDefaults | _debug(workflowArgs, "processed") // run workflow | innerWorkflowFactory(workflowArgs) - // check output tuple - | map { id_, output_ -> + def chInitialOutputList = chInitialOutputMulti instanceof List ? chInitialOutputMulti : [chInitialOutputMulti] + assert chInitialOutputList.size() > 0: "should have emitted at least one output channel" + // Add a channel ID to the events, which designates the channel the event was emitted from as a running number + // This number is used to sort the events later when the events are gathered from across the channels. + def chInitialOutputListWithIndexedEvents = chInitialOutputList.withIndex().collect{channel, channelIndex -> + def newChannel = channel + | map {tuple -> + assert tuple instanceof List : + "Error in module '${key_}': element in output channel should be a tuple [id, data, ...otherargs...]\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Expected class: List. Found: tuple.getClass() is ${tuple.getClass()}" + + def newEvent = [channelIndex] + tuple + return newEvent + } + return newChannel + } + // Put the events into 1 channel, cover case where there is only one channel is emitted + def chInitialOutput = chInitialOutputList.size() > 1 ? \ + chInitialOutputListWithIndexedEvents[0].mix(*chInitialOutputListWithIndexedEvents.tail()) : \ + chInitialOutputListWithIndexedEvents[0] + def chInitialOutputProcessed = chInitialOutput + | map { tuple -> + def channelId = tuple[0] + def id_ = tuple[1] + def output_ = tuple[2] // see if output map contains metadata def meta_ = @@ -2734,19 +2887,94 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { output_ = output_.findAll{k, v -> k != "_meta"} // check value types - output_ = _processOutputValues(output_, meta.config, id_, key_) + output_ = _checkValidOutputArgument(output_, meta.config, id_, key_) - // simplify output if need be - if (workflowArgs.auto.simplifyOutput && output_.size() == 1) { - output_ = output_.values()[0] - } - - [join_id, id_, output_] + [join_id, channelId, id_, output_] } // | view{"chInitialOutput: ${it.take(3)}"} + // join the output [prev_id, channel_id, new_id, output] with the previous state [prev_id, state, ...] + def chPublishWithPreviousState = safeJoin(chInitialOutputProcessed, chRunFiltered, key_) + // input tuple format: [join_id, channel_id, id, output, prev_state, ...] + // output tuple format: [join_id, channel_id, id, new_state, ...] + | map{ tup -> + def new_state = workflowArgs.toState(tup.drop(2).take(3)) + tup.take(3) + [new_state] + tup.drop(5) + } + if (workflowArgs.auto.publish == "state") { + def chPublishFiles = chPublishWithPreviousState + // input tuple format: [join_id, channel_id, id, new_state, ...] + // output tuple format: [join_id, channel_id, id, new_state] + | map{ tup -> + tup.take(4) + } + + safeJoin(chPublishFiles, chArgsWithDefaults, key_) + // input tuple format: [join_id, channel_id, id, new_state, orig_state, ...] + // output tuple format: [id, new_state, orig_state] + | map { tup -> + tup.drop(2).take(3) + } + | publishFilesByConfig(key: key_, config: meta.config) + } + // Join the state from the events that were emitted from different channels + def chJoined = chInitialOutputProcessed + | map {tuple -> + def join_id = tuple[0] + def channel_id = tuple[1] + def id = tuple[2] + def other = tuple.drop(3) + // Below, groupTuple is used to join the events. To make sure resuming a workflow + // keeps working, the output state must be deterministic. This means the state needs to be + // sorted with groupTuple's has a 'sort' argument. This argument can be set to 'hash', + // but hashing the state when it is large can be problematic in terms of performance. + // Therefore, a custom comparator function is provided. We add the channel ID to the + // states so that we can use the channel ID to sort the items. + def stateWithChannelID = [[channel_id] * other.size(), other].transpose() + // A comparator that is provided to groupTuple's 'sort' argument is applied + // to all elements of the event tuple (that is not the 'id'). The comparator + // closure that is used below expects the input to be List. So the join_id and + // channel_id must also be wrapped in a list. + [[join_id], [channel_id], id] + stateWithChannelID + } + | groupTuple(by: 2, sort: {a, b -> a[0] <=> b[0]}, size: chInitialOutputList.size(), remainder: true) + | map {join_ids, _, id, statesWithChannelID -> + // Remove the channel IDs from the states + def states = statesWithChannelID.collect{it[1]} + def newJoinId = join_ids.flatten().unique{a, b -> a <=> b} + assert newJoinId.size() == 1: "Multiple events were emitted for '$id'." + def newJoinIdUnique = newJoinId[0] + + // Merge the states from the different channels + def newState = states.inject([:]){ old_state, state_to_add -> + return old_state + state_to_add.collectEntries{k, v -> + if (!multipleArgs.contains(k)) { + // if the key is not a multiple argument, we expect only one value + if (old_state.containsKey(k)) { + assert old_state[k] == v : "ID $id: multiple entries for argument $k were emitted." + } + [k, v] + } else { + // if the key is a multiple argument, append the different values into one list + def prevValue = old_state.getOrDefault(k, []) + def prevValueAsList = prevValue instanceof List ? prevValue : [prevValue] + [k, prevValueAsList + v] + } + } + } + + _checkAllRequiredOuputsPresent(newState, meta.config, id, key_) + + // simplify output if need be + if (workflowArgs.auto.simplifyOutput && newState.size() == 1) { + newState = newState.values()[0] + } + + return [newJoinIdUnique, id, newState] + } + // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) + def chNewState = safeJoin(chJoined, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -2755,23 +2983,21 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { } if (workflowArgs.auto.publish == "state") { - def chPublish = chNewState + def chPublishStates = chNewState // input tuple format: [join_id, id, new_state, ...] // output tuple format: [join_id, id, new_state] | map{ tup -> tup.take(3) } - safeJoin(chPublish, chArgsWithDefaults, key_) + safeJoin(chPublishStates, chArgsWithDefaults, key_) // input tuple format: [join_id, id, new_state, orig_state, ...] // output tuple format: [id, new_state, orig_state] | map { tup -> tup.drop(1).take(3) - } + } | publishStatesByConfig(key: key_, config: meta.config) } - - // remove join_id and meta chReturn = chNewState | map { tup -> // input tuple format: [join_id, id, new_state, ...] @@ -2806,7 +3032,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "publish", "namespace" : "io", - "version" : "v0.3.8", + "version" : "v0.3.9", "argument_groups" : [ { "name" : "Input arguments", @@ -2929,6 +3155,10 @@ meta = [ ], "description" : "Publish the processed results of the run", "status" : "enabled", + "scope" : { + "image" : "public", + "target" : "public" + }, "requirements" : { "commands" : [ "ps" @@ -3021,7 +3251,7 @@ meta = [ "id" : "docker", "image" : "debian:stable-slim", "target_registry" : "images.viash-hub.com", - "target_tag" : "v0.3.8", + "target_tag" : "v0.3.9", "namespace_separator" : "/", "setup" : [ { @@ -3043,14 +3273,14 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/io/publish", - "viash_version" : "0.9.0", - "git_commit" : "dafe2d829079bd5a9f5eb4e7cf63e92edbf1a742", + "viash_version" : "0.9.4", + "git_commit" : "820318c378d8119e2aa98768282e43c2aa017ba7", "git_remote" : "https://github.com/viash-hub/demultiplex", - "git_tag" : "v0.3.5-9-gdafe2d8" + "git_tag" : "v0.3.8-5-g820318c" }, "package_config" : { "name" : "demultiplex", - "version" : "v0.3.8", + "version" : "v0.3.9", "description" : "Demultiplexing pipeline\n", "info" : { "test_resources" : [ @@ -3060,14 +3290,14 @@ meta = [ } ] }, - "viash_version" : "0.9.0", + "viash_version" : "0.9.4", "source" : "src", "target" : "target", "config_mods" : [ - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n", + ".requirements.commands += ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v0.3.8'" + ".engines[.type == 'docker'].target_tag := 'v0.3.9'" ], "keywords" : [ "bioinformatics", @@ -3491,7 +3721,7 @@ def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) { // create process from temp file def binding = new nextflow.script.ScriptBinding([:]) def session = nextflow.Nextflow.getSession() - def parser = new nextflow.script.ScriptParser(session) + def parser = _getScriptLoader(session) .setModule(true) .setBinding(binding) def moduleScript = parser.runScript(tempFile) @@ -3505,6 +3735,27 @@ def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) { return scriptMeta.getProcess(procKey) } +// use Reflection to get a ScriptParser / ScriptLoader +// <25.02.0-edge: new nextflow.script.ScriptParser(session) +// >=25.02.0-edge: nextflow.script.ScriptLoaderFactory.create(session) +def _getScriptLoader(nextflow.Session session) { + // try using the old method + try { + Class scriptParserClass = Class.forName('nextflow.script.ScriptParser') + return scriptParserClass.getDeclaredConstructor(nextflow.Session).newInstance(session) + } catch (ClassNotFoundException e) { + // else try with the new method + try { + Class scriptLoaderFactoryClass = Class.forName('nextflow.script.ScriptLoaderFactory') + def createMethod = scriptLoaderFactoryClass.getDeclaredMethod('create', nextflow.Session) + return createMethod.invoke(null, session) // null because create is static + } catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException | java.lang.reflect.InvocationTargetException e2) { + // Handle the case where neither class is found + throw new Exception("Neither nextflow.script.ScriptParser nor nextflow.script.ScriptLoaderFactory could be found. Is this a compatible Nextflow version?", e2) + } + } +} + // defaults meta["defaults"] = [ // key to be used to trace the process and determine output names @@ -3518,7 +3769,7 @@ meta["defaults"] = [ "container" : { "registry" : "images.viash-hub.com", "image" : "vsh/demultiplex/io/publish", - "tag" : "v0.3.8" + "tag" : "v0.3.9" }, "tag" : "$id" }'''), diff --git a/target/nextflow/io/publish/nextflow.config b/target/nextflow/io/publish/nextflow.config index 001a4bb..4d943db 100644 --- a/target/nextflow/io/publish/nextflow.config +++ b/target/nextflow/io/publish/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'io/publish' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v0.3.8' + version = 'v0.3.9' description = 'Publish the processed results of the run' } diff --git a/target/nextflow/io/publish/nextflow_schema.json b/target/nextflow/io/publish/nextflow_schema.json index d5483f3..7a2bb38 100644 --- a/target/nextflow/io/publish/nextflow_schema.json +++ b/target/nextflow/io/publish/nextflow_schema.json @@ -67,10 +67,10 @@ "output": { "type": "string", - "description": "Type: `file`, default: `$id.$key.output.output`. ", - "help_text": "Type: `file`, default: `$id.$key.output.output`. " + "description": "Type: `file`, default: `fastq`. ", + "help_text": "Type: `file`, default: `fastq`. " , - "default":"$id.$key.output.output" + "default":"fastq" } @@ -78,10 +78,10 @@ "output_falco": { "type": "string", - "description": "Type: `file`, default: `$id.$key.output_falco.output_falco`. ", - "help_text": "Type: `file`, default: `$id.$key.output_falco.output_falco`. " + "description": "Type: `file`, default: `qc/fastqc`. ", + "help_text": "Type: `file`, default: `qc/fastqc`. " , - "default":"$id.$key.output_falco.output_falco" + "default":"qc/fastqc" } @@ -89,10 +89,10 @@ "output_multiqc": { "type": "string", - "description": "Type: `file`, default: `$id.$key.output_multiqc.html`. ", - "help_text": "Type: `file`, default: `$id.$key.output_multiqc.html`. " + "description": "Type: `file`, default: `qc/multiqc_report.html`. ", + "help_text": "Type: `file`, default: `qc/multiqc_report.html`. " , - "default":"$id.$key.output_multiqc.html" + "default":"qc/multiqc_report.html" } @@ -100,10 +100,10 @@ "output_run_information": { "type": "string", - "description": "Type: `file`, default: `$id.$key.output_run_information.csv`. ", - "help_text": "Type: `file`, default: `$id.$key.output_run_information.csv`. " + "description": "Type: `file`, default: `run_information.csv`. ", + "help_text": "Type: `file`, default: `run_information.csv`. " , - "default":"$id.$key.output_run_information.csv" + "default":"run_information.csv" } diff --git a/target/nextflow/io/untar/.config.vsh.yaml b/target/nextflow/io/untar/.config.vsh.yaml index d533e2a..48eaaf4 100644 --- a/target/nextflow/io/untar/.config.vsh.yaml +++ b/target/nextflow/io/untar/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "untar" namespace: "io" -version: "v0.3.8" +version: "v0.3.9" argument_groups: - name: "Input arguments" arguments: @@ -57,6 +57,9 @@ test_resources: is_executable: true info: null status: "enabled" +scope: + image: "public" + target: "public" requirements: commands: - "ps" @@ -135,7 +138,7 @@ engines: id: "docker" image: "debian:stable-slim" target_registry: "images.viash-hub.com" - target_tag: "v0.3.8" + target_tag: "v0.3.9" namespace_separator: "/" setup: - type: "apt" @@ -152,29 +155,29 @@ build_info: engine: "docker|native" output: "target/nextflow/io/untar" executable: "target/nextflow/io/untar/main.nf" - viash_version: "0.9.0" - git_commit: "dafe2d829079bd5a9f5eb4e7cf63e92edbf1a742" + viash_version: "0.9.4" + git_commit: "820318c378d8119e2aa98768282e43c2aa017ba7" git_remote: "https://github.com/viash-hub/demultiplex" - git_tag: "v0.3.5-9-gdafe2d8" + git_tag: "v0.3.8-5-g820318c" package_config: name: "demultiplex" - version: "v0.3.8" + version: "v0.3.9" description: "Demultiplexing pipeline\n" info: test_resources: - path: "gs://viash-hub-test-data/demultiplex/v2/" dest: "testData" - viash_version: "0.9.0" + viash_version: "0.9.4" source: "src" target: "target" config_mods: - - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\ + - ".requirements.commands += ['ps']\n.runners[.type == 'nextflow'].directives.tag\ \ := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n\ .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ )'\n" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.3.8'" + - ".engines[.type == 'docker'].target_tag := 'v0.3.9'" keywords: - "bioinformatics" - "sequence" diff --git a/target/nextflow/io/untar/main.nf b/target/nextflow/io/untar/main.nf index 51af0e0..842a85c 100644 --- a/target/nextflow/io/untar/main.nf +++ b/target/nextflow/io/untar/main.nf @@ -1,6 +1,6 @@ -// untar v0.3.8 +// untar v0.3.9 // -// This wrapper script is auto-generated by viash 0.9.0 and is thus a derivative +// This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data // Intuitive. // @@ -82,64 +82,56 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi foundClass = "List[${e.foundClass}]" } } else if (par.type == "string") { - // cast to string if need be + // cast to string if need be. only cast if the value is a GString if (value instanceof GString) { - value = value.toString() + value = value as String } expectedClass = value instanceof String ? null : "String" } else if (par.type == "integer") { // cast to integer if need be - if (value instanceof String) { + if (value !instanceof Integer) { try { - value = value.toInteger() + value = value as Integer } catch (NumberFormatException e) { - // do nothing + expectedClass = "Integer" } } - if (value instanceof java.math.BigInteger) { - value = value.intValue() - } - expectedClass = value instanceof Integer ? null : "Integer" } else if (par.type == "long") { // cast to long if need be - if (value instanceof String) { + if (value !instanceof Long) { try { - value = value.toLong() + value = value as Long } catch (NumberFormatException e) { - // do nothing + expectedClass = "Long" } } - if (value instanceof Integer) { - value = value.toLong() - } - expectedClass = value instanceof Long ? null : "Long" } else if (par.type == "double") { // cast to double if need be - if (value instanceof String) { + if (value !instanceof Double) { try { - value = value.toDouble() + value = value as Double } catch (NumberFormatException e) { - // do nothing + expectedClass = "Double" } } - if (value instanceof java.math.BigDecimal) { - value = value.doubleValue() + } else if (par.type == "float") { + // cast to float if need be + if (value !instanceof Float) { + try { + value = value as Float + } catch (NumberFormatException e) { + expectedClass = "Float" + } } - if (value instanceof Float) { - value = value.toDouble() - } - expectedClass = value instanceof Double ? null : "Double" } else if (par.type == "boolean" | par.type == "boolean_true" | par.type == "boolean_false") { // cast to boolean if need be - if (value instanceof String) { - def valueLower = value.toLowerCase() - if (valueLower == "true") { - value = true - } else if (valueLower == "false") { - value = false + if (value !instanceof Boolean) { + try { + value = value as Boolean + } catch (Exception e) { + expectedClass = "Boolean" } } - expectedClass = value instanceof Boolean ? null : "Boolean" } else if (par.type == "file" && (par.direction == "input" || stage == "output")) { // cast to path if need be if (value instanceof String) { @@ -151,10 +143,13 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi expectedClass = value instanceof Path ? null : "Path" } else if (par.type == "file" && stage == "input" && par.direction == "output") { // cast to string if need be - if (value instanceof GString) { - value = value.toString() + if (value !instanceof String) { + try { + value = value as String + } catch (Exception e) { + expectedClass = "String" + } } - expectedClass = value instanceof String ? null : "String" } else { // didn't find a match for par.type expectedClass = par.type @@ -173,7 +168,7 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi Map _processInputValues(Map inputs, Map config, String id, String key) { if (!workflow.stubRun) { config.allArguments.each { arg -> - if (arg.required) { + if (arg.required && arg.direction == "input") { assert inputs.containsKey(arg.plainName) && inputs.get(arg.plainName) != null : "Error in module '${key}' id '${id}': required input argument '${arg.plainName}' is missing" } @@ -192,15 +187,8 @@ Map _processInputValues(Map inputs, Map config, String id, String key) { } // helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_processOutputValues.nf' -Map _processOutputValues(Map outputs, Map config, String id, String key) { +Map _checkValidOutputArgument(Map outputs, Map config, String id, String key) { if (!workflow.stubRun) { - config.allArguments.each { arg -> - if (arg.direction == "output" && arg.required) { - assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : - "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing" - } - } - outputs = outputs.collectEntries { name, value -> def par = config.allArguments.find { it.plainName == name && it.direction == "output" } assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid output argument" @@ -213,6 +201,16 @@ Map _processOutputValues(Map outputs, Map config, String id, String key) { return outputs } +void _checkAllRequiredOuputsPresent(Map outputs, Map config, String id, String key) { + if (!workflow.stubRun) { + config.allArguments.each { arg -> + if (arg.direction == "output" && arg.required) { + assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : + "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing" + } + } + } +} // helper file: 'src/main/resources/io/viash/runners/nextflow/channel/IDChecker.nf' class IDChecker { final def items = [] as Set @@ -1666,6 +1664,162 @@ def joinStates(Closure apply_) { } return joinStatesWf } +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishFiles.nf' +def publishFiles(Map args) { + def key_ = args.get("key") + + assert key_ != null : "publishFiles: key must be specified" + + workflow publishFilesWf { + take: input_ch + main: + input_ch + | map { tup -> + def id_ = tup[0] + def state_ = tup[1] + + // the input files and the target output filenames + def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose() + def inputFiles_ = inputoutputFilenames_[0] + def outputFilenames_ = inputoutputFilenames_[1] + + [id_, inputFiles_, outputFilenames_] + } + | publishFilesProc + emit: input_ch + } + return publishFilesWf +} + +process publishFilesProc { + // todo: check publishpath? + publishDir path: "${getPublishDir()}/", mode: "copy" + tag "$id" + input: + tuple val(id), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles) + output: + tuple val(id), path{outputFiles} + script: + def copyCommands = [ + inputFiles instanceof List ? inputFiles : [inputFiles], + outputFiles instanceof List ? outputFiles : [outputFiles] + ] + .transpose() + .collectMany{infile, outfile -> + if (infile.toString() != outfile.toString()) { + [ + "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"", + "cp -r '${infile.toString()}' '${outfile.toString()}'" + ] + } else { + // no need to copy if infile is the same as outfile + [] + } + } + """ + echo "Copying output files to destination folder" + ${copyCommands.join("\n ")} + """ +} + + +// this assumes that the state contains no other values other than those specified in the config +def publishFilesByConfig(Map args) { + def config = args.get("config") + assert config != null : "publishFilesByConfig: config must be specified" + + def key_ = args.get("key", config.name) + assert key_ != null : "publishFilesByConfig: key must be specified" + + workflow publishFilesSimpleWf { + take: input_ch + main: + input_ch + | map { tup -> + def id_ = tup[0] + def state_ = tup[1] // e.g. [output: new File("myoutput.h5ad"), k: 10] + def origState_ = tup[2] // e.g. [output: '$id.$key.foo.h5ad'] + + + // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where + // - key is a String + // - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path) + // - inputPath is a List[Path] + // - outputFilename is a List[String] + // - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml) + def processedState = + config.allArguments + .findAll { it.direction == "output" } + .collectMany { par -> + def plainName_ = par.plainName + // if the state does not contain the key, it's an + // optional argument for which the component did + // not generate any output OR multiple channels were emitted + // and the output was just not added to using the channel + // that is now being parsed + if (!state_.containsKey(plainName_)) { + return [] + } + def value = state_[plainName_] + // if the parameter is not a file, it should be stored + // in the state as-is, but is not something that needs + // to be copied from the source path to the dest path + if (par.type != "file") { + return [[inputPath: [], outputFilename: []]] + } + // if the orig state does not contain this filename, + // it's an optional argument for which the user specified + // that it should not be returned as a state + if (!origState_.containsKey(plainName_)) { + return [] + } + def filenameTemplate = origState_[plainName_] + // if the pararameter is multiple: true, fetch the template + if (par.multiple && filenameTemplate instanceof List) { + filenameTemplate = filenameTemplate[0] + } + // instantiate the template + def filename = filenameTemplate + .replaceAll('\\$id', id_) + .replaceAll('\\$\\{id\\}', id_) + .replaceAll('\\$key', key_) + .replaceAll('\\$\\{key\\}', key_) + if (par.multiple) { + // if the parameter is multiple: true, the filename + // should contain a wildcard '*' that is replaced with + // the index of the file + assert filename.contains("*") : "Module '${key_}' id '${id_}': Multiple output files specified, but no wildcard '*' in the filename: ${filename}" + def outputPerFile = value.withIndex().collect{ val, ix -> + def filename_ix = filename.replace("*", ix.toString()) + def inputPath = val instanceof File ? val.toPath() : val + [inputPath: inputPath, outputFilename: filename_ix] + } + def transposedOutputs = ["inputPath", "outputFilename"].collectEntries{ key -> + [key, outputPerFile.collect{dic -> dic[key]}] + } + return [[key: plainName_] + transposedOutputs] + } else { + def value_ = java.nio.file.Paths.get(filename) + def inputPath = value instanceof File ? value.toPath() : value + return [[inputPath: [inputPath], outputFilename: [filename]]] + } + } + + def inputPaths = processedState.collectMany{it.inputPath} + def outputFilenames = processedState.collectMany{it.outputFilename} + + + [id_, inputPaths, outputFilenames] + } + | publishFilesProc + emit: input_ch + } + return publishFilesSimpleWf +} + + + + // helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishStates.nf' def collectFiles(obj) { if (obj instanceof java.io.File || obj instanceof Path) { @@ -1723,8 +1877,6 @@ def publishStates(Map args) { // the input files and the target output filenames def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose() - def inputFiles_ = inputoutputFilenames_[0] - def outputFilenames_ = inputoutputFilenames_[1] def yamlFilename = yamlTemplate_ .replaceAll('\\$id', id_) @@ -1737,7 +1889,7 @@ def publishStates(Map args) { // convert state to yaml blob def yamlBlob_ = toRelativeTaggedYamlBlob([id: id_] + state_, java.nio.file.Paths.get(yamlFilename)) - [id_, yamlBlob_, yamlFilename, inputFiles_, outputFilenames_] + [id_, yamlBlob_, yamlFilename] } | publishStatesProc emit: input_ch @@ -1749,33 +1901,17 @@ process publishStatesProc { publishDir path: "${getPublishDir()}/", mode: "copy" tag "$id" input: - tuple val(id), val(yamlBlob), val(yamlFile), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles) + tuple val(id), val(yamlBlob), val(yamlFile) output: - tuple val(id), path{[yamlFile] + outputFiles} + tuple val(id), path{[yamlFile]} script: - def copyCommands = [ - inputFiles instanceof List ? inputFiles : [inputFiles], - outputFiles instanceof List ? outputFiles : [outputFiles] - ] - .transpose() - .collectMany{infile, outfile -> - if (infile.toString() != outfile.toString()) { - [ - "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"", - "cp -r '${infile.toString()}' '${outfile.toString()}'" - ] - } else { - // no need to copy if infile is the same as outfile - [] - } - } """ -mkdir -p "\$(dirname '${yamlFile}')" -echo "Storing state as yaml" -echo '${yamlBlob}' > '${yamlFile}' -echo "Copying output files to destination folder" -${copyCommands.join("\n ")} -""" + mkdir -p "\$(dirname '${yamlFile}')" + echo "Storing state as yaml" + cat > '${yamlFile}' << HERE +${yamlBlob} +HERE + """ } @@ -1806,13 +1942,10 @@ def publishStatesByConfig(Map args) { .replaceAll('\\$\\{key\\}', key_) def yamlDir = java.nio.file.Paths.get(yamlFilename).getParent() - // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where + // the processed state is a list of [key, value] tuples, where // - key is a String // - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path) - // - inputPath is a List[Path] - // - outputFilename is a List[String] // - (key, value) are the tuples that will be saved to the state.yaml file - // - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml) def processedState = config.allArguments .findAll { it.direction == "output" } @@ -1829,7 +1962,7 @@ def publishStatesByConfig(Map args) { // in the state as-is, but is not something that needs // to be copied from the source path to the dest path if (par.type != "file") { - return [[key: plainName_, value: value, inputPath: [], outputFilename: []]] + return [[key: plainName_, value: value]] } // if the orig state does not contain this filename, // it's an optional argument for which the user specified @@ -1860,13 +1993,9 @@ def publishStatesByConfig(Map args) { if (yamlDir != null) { value_ = yamlDir.relativize(value_) } - def inputPath = val instanceof File ? val.toPath() : val - [value: value_, inputPath: inputPath, outputFilename: filename_ix] + return value_ } - def transposedOutputs = ["value", "inputPath", "outputFilename"].collectEntries{ key -> - [key, outputPerFile.collect{dic -> dic[key]}] - } - return [[key: plainName_] + transposedOutputs] + return [["key": plainName_, "value": outputPerFile]] } else { def value_ = java.nio.file.Paths.get(filename) // if id contains a slash @@ -1874,18 +2003,17 @@ def publishStatesByConfig(Map args) { value_ = yamlDir.relativize(value_) } def inputPath = value instanceof File ? value.toPath() : value - return [[key: plainName_, value: value_, inputPath: [inputPath], outputFilename: [filename]]] + return [["key": plainName_, value: value_]] } } + def updatedState_ = processedState.collectEntries{[it.key, it.value]} - def inputPaths = processedState.collectMany{it.inputPath} - def outputFilenames = processedState.collectMany{it.outputFilename} // convert state to yaml blob def yamlBlob_ = toTaggedYamlBlob([id: id_] + updatedState_) - [id_, yamlBlob_, yamlFilename, inputPaths, outputFilenames] + [id_, yamlBlob_, yamlFilename] } | publishStatesProc emit: input_ch @@ -2559,7 +2687,8 @@ def _debug(workflowArgs, debugKey) { def workflowFactory(Map args, Map defaultWfArgs, Map meta) { def workflowArgs = processWorkflowArgs(args, defaultWfArgs, meta) def key_ = workflowArgs["key"] - + def multipleArgs = meta.config.allArguments.findAll{ it.multiple }.collect{it.plainName} + workflow workflowInstance { take: input_ @@ -2716,12 +2845,36 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { } // TODO: move some of the _meta.join_id wrangling to the safeJoin() function. - def chInitialOutput = chArgsWithDefaults + def chInitialOutputMulti = chArgsWithDefaults | _debug(workflowArgs, "processed") // run workflow | innerWorkflowFactory(workflowArgs) - // check output tuple - | map { id_, output_ -> + def chInitialOutputList = chInitialOutputMulti instanceof List ? chInitialOutputMulti : [chInitialOutputMulti] + assert chInitialOutputList.size() > 0: "should have emitted at least one output channel" + // Add a channel ID to the events, which designates the channel the event was emitted from as a running number + // This number is used to sort the events later when the events are gathered from across the channels. + def chInitialOutputListWithIndexedEvents = chInitialOutputList.withIndex().collect{channel, channelIndex -> + def newChannel = channel + | map {tuple -> + assert tuple instanceof List : + "Error in module '${key_}': element in output channel should be a tuple [id, data, ...otherargs...]\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Expected class: List. Found: tuple.getClass() is ${tuple.getClass()}" + + def newEvent = [channelIndex] + tuple + return newEvent + } + return newChannel + } + // Put the events into 1 channel, cover case where there is only one channel is emitted + def chInitialOutput = chInitialOutputList.size() > 1 ? \ + chInitialOutputListWithIndexedEvents[0].mix(*chInitialOutputListWithIndexedEvents.tail()) : \ + chInitialOutputListWithIndexedEvents[0] + def chInitialOutputProcessed = chInitialOutput + | map { tuple -> + def channelId = tuple[0] + def id_ = tuple[1] + def output_ = tuple[2] // see if output map contains metadata def meta_ = @@ -2734,19 +2887,94 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { output_ = output_.findAll{k, v -> k != "_meta"} // check value types - output_ = _processOutputValues(output_, meta.config, id_, key_) + output_ = _checkValidOutputArgument(output_, meta.config, id_, key_) - // simplify output if need be - if (workflowArgs.auto.simplifyOutput && output_.size() == 1) { - output_ = output_.values()[0] - } - - [join_id, id_, output_] + [join_id, channelId, id_, output_] } // | view{"chInitialOutput: ${it.take(3)}"} + // join the output [prev_id, channel_id, new_id, output] with the previous state [prev_id, state, ...] + def chPublishWithPreviousState = safeJoin(chInitialOutputProcessed, chRunFiltered, key_) + // input tuple format: [join_id, channel_id, id, output, prev_state, ...] + // output tuple format: [join_id, channel_id, id, new_state, ...] + | map{ tup -> + def new_state = workflowArgs.toState(tup.drop(2).take(3)) + tup.take(3) + [new_state] + tup.drop(5) + } + if (workflowArgs.auto.publish == "state") { + def chPublishFiles = chPublishWithPreviousState + // input tuple format: [join_id, channel_id, id, new_state, ...] + // output tuple format: [join_id, channel_id, id, new_state] + | map{ tup -> + tup.take(4) + } + + safeJoin(chPublishFiles, chArgsWithDefaults, key_) + // input tuple format: [join_id, channel_id, id, new_state, orig_state, ...] + // output tuple format: [id, new_state, orig_state] + | map { tup -> + tup.drop(2).take(3) + } + | publishFilesByConfig(key: key_, config: meta.config) + } + // Join the state from the events that were emitted from different channels + def chJoined = chInitialOutputProcessed + | map {tuple -> + def join_id = tuple[0] + def channel_id = tuple[1] + def id = tuple[2] + def other = tuple.drop(3) + // Below, groupTuple is used to join the events. To make sure resuming a workflow + // keeps working, the output state must be deterministic. This means the state needs to be + // sorted with groupTuple's has a 'sort' argument. This argument can be set to 'hash', + // but hashing the state when it is large can be problematic in terms of performance. + // Therefore, a custom comparator function is provided. We add the channel ID to the + // states so that we can use the channel ID to sort the items. + def stateWithChannelID = [[channel_id] * other.size(), other].transpose() + // A comparator that is provided to groupTuple's 'sort' argument is applied + // to all elements of the event tuple (that is not the 'id'). The comparator + // closure that is used below expects the input to be List. So the join_id and + // channel_id must also be wrapped in a list. + [[join_id], [channel_id], id] + stateWithChannelID + } + | groupTuple(by: 2, sort: {a, b -> a[0] <=> b[0]}, size: chInitialOutputList.size(), remainder: true) + | map {join_ids, _, id, statesWithChannelID -> + // Remove the channel IDs from the states + def states = statesWithChannelID.collect{it[1]} + def newJoinId = join_ids.flatten().unique{a, b -> a <=> b} + assert newJoinId.size() == 1: "Multiple events were emitted for '$id'." + def newJoinIdUnique = newJoinId[0] + + // Merge the states from the different channels + def newState = states.inject([:]){ old_state, state_to_add -> + return old_state + state_to_add.collectEntries{k, v -> + if (!multipleArgs.contains(k)) { + // if the key is not a multiple argument, we expect only one value + if (old_state.containsKey(k)) { + assert old_state[k] == v : "ID $id: multiple entries for argument $k were emitted." + } + [k, v] + } else { + // if the key is a multiple argument, append the different values into one list + def prevValue = old_state.getOrDefault(k, []) + def prevValueAsList = prevValue instanceof List ? prevValue : [prevValue] + [k, prevValueAsList + v] + } + } + } + + _checkAllRequiredOuputsPresent(newState, meta.config, id, key_) + + // simplify output if need be + if (workflowArgs.auto.simplifyOutput && newState.size() == 1) { + newState = newState.values()[0] + } + + return [newJoinIdUnique, id, newState] + } + // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) + def chNewState = safeJoin(chJoined, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -2755,23 +2983,21 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { } if (workflowArgs.auto.publish == "state") { - def chPublish = chNewState + def chPublishStates = chNewState // input tuple format: [join_id, id, new_state, ...] // output tuple format: [join_id, id, new_state] | map{ tup -> tup.take(3) } - safeJoin(chPublish, chArgsWithDefaults, key_) + safeJoin(chPublishStates, chArgsWithDefaults, key_) // input tuple format: [join_id, id, new_state, orig_state, ...] // output tuple format: [id, new_state, orig_state] | map { tup -> tup.drop(1).take(3) - } + } | publishStatesByConfig(key: key_, config: meta.config) } - - // remove join_id and meta chReturn = chNewState | map { tup -> // input tuple format: [join_id, id, new_state, ...] @@ -2806,7 +3032,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "untar", "namespace" : "io", - "version" : "v0.3.8", + "version" : "v0.3.9", "argument_groups" : [ { "name" : "Input arguments", @@ -2882,6 +3108,10 @@ meta = [ } ], "status" : "enabled", + "scope" : { + "image" : "public", + "target" : "public" + }, "requirements" : { "commands" : [ "ps" @@ -2974,7 +3204,7 @@ meta = [ "id" : "docker", "image" : "debian:stable-slim", "target_registry" : "images.viash-hub.com", - "target_tag" : "v0.3.8", + "target_tag" : "v0.3.9", "namespace_separator" : "/", "setup" : [ { @@ -2996,14 +3226,14 @@ meta = [ "runner" : "nextflow", "engine" : "docker|native", "output" : "target/nextflow/io/untar", - "viash_version" : "0.9.0", - "git_commit" : "dafe2d829079bd5a9f5eb4e7cf63e92edbf1a742", + "viash_version" : "0.9.4", + "git_commit" : "820318c378d8119e2aa98768282e43c2aa017ba7", "git_remote" : "https://github.com/viash-hub/demultiplex", - "git_tag" : "v0.3.5-9-gdafe2d8" + "git_tag" : "v0.3.8-5-g820318c" }, "package_config" : { "name" : "demultiplex", - "version" : "v0.3.8", + "version" : "v0.3.9", "description" : "Demultiplexing pipeline\n", "info" : { "test_resources" : [ @@ -3013,14 +3243,14 @@ meta = [ } ] }, - "viash_version" : "0.9.0", + "viash_version" : "0.9.4", "source" : "src", "target" : "target", "config_mods" : [ - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n", + ".requirements.commands += ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v0.3.8'" + ".engines[.type == 'docker'].target_tag := 'v0.3.9'" ], "keywords" : [ "bioinformatics", @@ -3446,7 +3676,7 @@ def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) { // create process from temp file def binding = new nextflow.script.ScriptBinding([:]) def session = nextflow.Nextflow.getSession() - def parser = new nextflow.script.ScriptParser(session) + def parser = _getScriptLoader(session) .setModule(true) .setBinding(binding) def moduleScript = parser.runScript(tempFile) @@ -3460,6 +3690,27 @@ def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) { return scriptMeta.getProcess(procKey) } +// use Reflection to get a ScriptParser / ScriptLoader +// <25.02.0-edge: new nextflow.script.ScriptParser(session) +// >=25.02.0-edge: nextflow.script.ScriptLoaderFactory.create(session) +def _getScriptLoader(nextflow.Session session) { + // try using the old method + try { + Class scriptParserClass = Class.forName('nextflow.script.ScriptParser') + return scriptParserClass.getDeclaredConstructor(nextflow.Session).newInstance(session) + } catch (ClassNotFoundException e) { + // else try with the new method + try { + Class scriptLoaderFactoryClass = Class.forName('nextflow.script.ScriptLoaderFactory') + def createMethod = scriptLoaderFactoryClass.getDeclaredMethod('create', nextflow.Session) + return createMethod.invoke(null, session) // null because create is static + } catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException | java.lang.reflect.InvocationTargetException e2) { + // Handle the case where neither class is found + throw new Exception("Neither nextflow.script.ScriptParser nor nextflow.script.ScriptLoaderFactory could be found. Is this a compatible Nextflow version?", e2) + } + } +} + // defaults meta["defaults"] = [ // key to be used to trace the process and determine output names @@ -3473,7 +3724,7 @@ meta["defaults"] = [ "container" : { "registry" : "images.viash-hub.com", "image" : "vsh/demultiplex/io/untar", - "tag" : "v0.3.8" + "tag" : "v0.3.9" }, "tag" : "$id" }'''), diff --git a/target/nextflow/io/untar/nextflow.config b/target/nextflow/io/untar/nextflow.config index 7e6db3d..0d3ec26 100644 --- a/target/nextflow/io/untar/nextflow.config +++ b/target/nextflow/io/untar/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'io/untar' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v0.3.8' + version = 'v0.3.9' description = 'Unpack a .tar file. When the contents of the .tar file is just a single directory,\nput the contents of the directory into the output folder instead of that directory.\n' } diff --git a/target/nextflow/io/untar/nextflow_schema.json b/target/nextflow/io/untar/nextflow_schema.json index 5363950..7ce31ff 100644 --- a/target/nextflow/io/untar/nextflow_schema.json +++ b/target/nextflow/io/untar/nextflow_schema.json @@ -37,10 +37,10 @@ "output": { "type": "string", - "description": "Type: `file`, required, default: `$id.$key.output.output`. Directory to write the contents of the ", - "help_text": "Type: `file`, required, default: `$id.$key.output.output`. Directory to write the contents of the .tar file to." + "description": "Type: `file`, required, default: `$id.$key.output`. Directory to write the contents of the ", + "help_text": "Type: `file`, required, default: `$id.$key.output`. Directory to write the contents of the .tar file to." , - "default":"$id.$key.output.output" + "default":"$id.$key.output" } diff --git a/target/nextflow/runner/.config.vsh.yaml b/target/nextflow/runner/.config.vsh.yaml index 4e7027f..9a4c977 100644 --- a/target/nextflow/runner/.config.vsh.yaml +++ b/target/nextflow/runner/.config.vsh.yaml @@ -1,5 +1,5 @@ name: "runner" -version: "v0.3.8" +version: "v0.3.9" argument_groups: - name: "Input arguments" arguments: @@ -103,6 +103,9 @@ resources: description: "Runner for demultiplexing of raw sequencing data" info: null status: "enabled" +scope: + image: "public" + target: "public" requirements: commands: - "ps" @@ -191,32 +194,32 @@ build_info: engine: "native|native" output: "target/nextflow/runner" executable: "target/nextflow/runner/main.nf" - viash_version: "0.9.0" - git_commit: "dafe2d829079bd5a9f5eb4e7cf63e92edbf1a742" + viash_version: "0.9.4" + git_commit: "820318c378d8119e2aa98768282e43c2aa017ba7" git_remote: "https://github.com/viash-hub/demultiplex" - git_tag: "v0.3.5-9-gdafe2d8" + git_tag: "v0.3.8-5-g820318c" dependencies: - "target/nextflow/demultiplex" - "target/nextflow/io/publish" package_config: name: "demultiplex" - version: "v0.3.8" + version: "v0.3.9" description: "Demultiplexing pipeline\n" info: test_resources: - path: "gs://viash-hub-test-data/demultiplex/v2/" dest: "testData" - viash_version: "0.9.0" + viash_version: "0.9.4" source: "src" target: "target" config_mods: - - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\ + - ".requirements.commands += ['ps']\n.runners[.type == 'nextflow'].directives.tag\ \ := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n\ .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ )'\n" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.3.8'" + - ".engines[.type == 'docker'].target_tag := 'v0.3.9'" keywords: - "bioinformatics" - "sequence" diff --git a/target/nextflow/runner/main.nf b/target/nextflow/runner/main.nf index 9e153d3..47cb6eb 100644 --- a/target/nextflow/runner/main.nf +++ b/target/nextflow/runner/main.nf @@ -1,6 +1,6 @@ -// runner v0.3.8 +// runner v0.3.9 // -// This wrapper script is auto-generated by viash 0.9.0 and is thus a derivative +// This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data // Intuitive. // @@ -82,64 +82,56 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi foundClass = "List[${e.foundClass}]" } } else if (par.type == "string") { - // cast to string if need be + // cast to string if need be. only cast if the value is a GString if (value instanceof GString) { - value = value.toString() + value = value as String } expectedClass = value instanceof String ? null : "String" } else if (par.type == "integer") { // cast to integer if need be - if (value instanceof String) { + if (value !instanceof Integer) { try { - value = value.toInteger() + value = value as Integer } catch (NumberFormatException e) { - // do nothing + expectedClass = "Integer" } } - if (value instanceof java.math.BigInteger) { - value = value.intValue() - } - expectedClass = value instanceof Integer ? null : "Integer" } else if (par.type == "long") { // cast to long if need be - if (value instanceof String) { + if (value !instanceof Long) { try { - value = value.toLong() + value = value as Long } catch (NumberFormatException e) { - // do nothing + expectedClass = "Long" } } - if (value instanceof Integer) { - value = value.toLong() - } - expectedClass = value instanceof Long ? null : "Long" } else if (par.type == "double") { // cast to double if need be - if (value instanceof String) { + if (value !instanceof Double) { try { - value = value.toDouble() + value = value as Double } catch (NumberFormatException e) { - // do nothing + expectedClass = "Double" } } - if (value instanceof java.math.BigDecimal) { - value = value.doubleValue() + } else if (par.type == "float") { + // cast to float if need be + if (value !instanceof Float) { + try { + value = value as Float + } catch (NumberFormatException e) { + expectedClass = "Float" + } } - if (value instanceof Float) { - value = value.toDouble() - } - expectedClass = value instanceof Double ? null : "Double" } else if (par.type == "boolean" | par.type == "boolean_true" | par.type == "boolean_false") { // cast to boolean if need be - if (value instanceof String) { - def valueLower = value.toLowerCase() - if (valueLower == "true") { - value = true - } else if (valueLower == "false") { - value = false + if (value !instanceof Boolean) { + try { + value = value as Boolean + } catch (Exception e) { + expectedClass = "Boolean" } } - expectedClass = value instanceof Boolean ? null : "Boolean" } else if (par.type == "file" && (par.direction == "input" || stage == "output")) { // cast to path if need be if (value instanceof String) { @@ -151,10 +143,13 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi expectedClass = value instanceof Path ? null : "Path" } else if (par.type == "file" && stage == "input" && par.direction == "output") { // cast to string if need be - if (value instanceof GString) { - value = value.toString() + if (value !instanceof String) { + try { + value = value as String + } catch (Exception e) { + expectedClass = "String" + } } - expectedClass = value instanceof String ? null : "String" } else { // didn't find a match for par.type expectedClass = par.type @@ -173,7 +168,7 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi Map _processInputValues(Map inputs, Map config, String id, String key) { if (!workflow.stubRun) { config.allArguments.each { arg -> - if (arg.required) { + if (arg.required && arg.direction == "input") { assert inputs.containsKey(arg.plainName) && inputs.get(arg.plainName) != null : "Error in module '${key}' id '${id}': required input argument '${arg.plainName}' is missing" } @@ -192,15 +187,8 @@ Map _processInputValues(Map inputs, Map config, String id, String key) { } // helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_processOutputValues.nf' -Map _processOutputValues(Map outputs, Map config, String id, String key) { +Map _checkValidOutputArgument(Map outputs, Map config, String id, String key) { if (!workflow.stubRun) { - config.allArguments.each { arg -> - if (arg.direction == "output" && arg.required) { - assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : - "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing" - } - } - outputs = outputs.collectEntries { name, value -> def par = config.allArguments.find { it.plainName == name && it.direction == "output" } assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid output argument" @@ -213,6 +201,16 @@ Map _processOutputValues(Map outputs, Map config, String id, String key) { return outputs } +void _checkAllRequiredOuputsPresent(Map outputs, Map config, String id, String key) { + if (!workflow.stubRun) { + config.allArguments.each { arg -> + if (arg.direction == "output" && arg.required) { + assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : + "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing" + } + } + } +} // helper file: 'src/main/resources/io/viash/runners/nextflow/channel/IDChecker.nf' class IDChecker { final def items = [] as Set @@ -1666,6 +1664,162 @@ def joinStates(Closure apply_) { } return joinStatesWf } +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishFiles.nf' +def publishFiles(Map args) { + def key_ = args.get("key") + + assert key_ != null : "publishFiles: key must be specified" + + workflow publishFilesWf { + take: input_ch + main: + input_ch + | map { tup -> + def id_ = tup[0] + def state_ = tup[1] + + // the input files and the target output filenames + def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose() + def inputFiles_ = inputoutputFilenames_[0] + def outputFilenames_ = inputoutputFilenames_[1] + + [id_, inputFiles_, outputFilenames_] + } + | publishFilesProc + emit: input_ch + } + return publishFilesWf +} + +process publishFilesProc { + // todo: check publishpath? + publishDir path: "${getPublishDir()}/", mode: "copy" + tag "$id" + input: + tuple val(id), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles) + output: + tuple val(id), path{outputFiles} + script: + def copyCommands = [ + inputFiles instanceof List ? inputFiles : [inputFiles], + outputFiles instanceof List ? outputFiles : [outputFiles] + ] + .transpose() + .collectMany{infile, outfile -> + if (infile.toString() != outfile.toString()) { + [ + "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"", + "cp -r '${infile.toString()}' '${outfile.toString()}'" + ] + } else { + // no need to copy if infile is the same as outfile + [] + } + } + """ + echo "Copying output files to destination folder" + ${copyCommands.join("\n ")} + """ +} + + +// this assumes that the state contains no other values other than those specified in the config +def publishFilesByConfig(Map args) { + def config = args.get("config") + assert config != null : "publishFilesByConfig: config must be specified" + + def key_ = args.get("key", config.name) + assert key_ != null : "publishFilesByConfig: key must be specified" + + workflow publishFilesSimpleWf { + take: input_ch + main: + input_ch + | map { tup -> + def id_ = tup[0] + def state_ = tup[1] // e.g. [output: new File("myoutput.h5ad"), k: 10] + def origState_ = tup[2] // e.g. [output: '$id.$key.foo.h5ad'] + + + // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where + // - key is a String + // - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path) + // - inputPath is a List[Path] + // - outputFilename is a List[String] + // - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml) + def processedState = + config.allArguments + .findAll { it.direction == "output" } + .collectMany { par -> + def plainName_ = par.plainName + // if the state does not contain the key, it's an + // optional argument for which the component did + // not generate any output OR multiple channels were emitted + // and the output was just not added to using the channel + // that is now being parsed + if (!state_.containsKey(plainName_)) { + return [] + } + def value = state_[plainName_] + // if the parameter is not a file, it should be stored + // in the state as-is, but is not something that needs + // to be copied from the source path to the dest path + if (par.type != "file") { + return [[inputPath: [], outputFilename: []]] + } + // if the orig state does not contain this filename, + // it's an optional argument for which the user specified + // that it should not be returned as a state + if (!origState_.containsKey(plainName_)) { + return [] + } + def filenameTemplate = origState_[plainName_] + // if the pararameter is multiple: true, fetch the template + if (par.multiple && filenameTemplate instanceof List) { + filenameTemplate = filenameTemplate[0] + } + // instantiate the template + def filename = filenameTemplate + .replaceAll('\\$id', id_) + .replaceAll('\\$\\{id\\}', id_) + .replaceAll('\\$key', key_) + .replaceAll('\\$\\{key\\}', key_) + if (par.multiple) { + // if the parameter is multiple: true, the filename + // should contain a wildcard '*' that is replaced with + // the index of the file + assert filename.contains("*") : "Module '${key_}' id '${id_}': Multiple output files specified, but no wildcard '*' in the filename: ${filename}" + def outputPerFile = value.withIndex().collect{ val, ix -> + def filename_ix = filename.replace("*", ix.toString()) + def inputPath = val instanceof File ? val.toPath() : val + [inputPath: inputPath, outputFilename: filename_ix] + } + def transposedOutputs = ["inputPath", "outputFilename"].collectEntries{ key -> + [key, outputPerFile.collect{dic -> dic[key]}] + } + return [[key: plainName_] + transposedOutputs] + } else { + def value_ = java.nio.file.Paths.get(filename) + def inputPath = value instanceof File ? value.toPath() : value + return [[inputPath: [inputPath], outputFilename: [filename]]] + } + } + + def inputPaths = processedState.collectMany{it.inputPath} + def outputFilenames = processedState.collectMany{it.outputFilename} + + + [id_, inputPaths, outputFilenames] + } + | publishFilesProc + emit: input_ch + } + return publishFilesSimpleWf +} + + + + // helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishStates.nf' def collectFiles(obj) { if (obj instanceof java.io.File || obj instanceof Path) { @@ -1723,8 +1877,6 @@ def publishStates(Map args) { // the input files and the target output filenames def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose() - def inputFiles_ = inputoutputFilenames_[0] - def outputFilenames_ = inputoutputFilenames_[1] def yamlFilename = yamlTemplate_ .replaceAll('\\$id', id_) @@ -1737,7 +1889,7 @@ def publishStates(Map args) { // convert state to yaml blob def yamlBlob_ = toRelativeTaggedYamlBlob([id: id_] + state_, java.nio.file.Paths.get(yamlFilename)) - [id_, yamlBlob_, yamlFilename, inputFiles_, outputFilenames_] + [id_, yamlBlob_, yamlFilename] } | publishStatesProc emit: input_ch @@ -1749,33 +1901,17 @@ process publishStatesProc { publishDir path: "${getPublishDir()}/", mode: "copy" tag "$id" input: - tuple val(id), val(yamlBlob), val(yamlFile), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles) + tuple val(id), val(yamlBlob), val(yamlFile) output: - tuple val(id), path{[yamlFile] + outputFiles} + tuple val(id), path{[yamlFile]} script: - def copyCommands = [ - inputFiles instanceof List ? inputFiles : [inputFiles], - outputFiles instanceof List ? outputFiles : [outputFiles] - ] - .transpose() - .collectMany{infile, outfile -> - if (infile.toString() != outfile.toString()) { - [ - "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"", - "cp -r '${infile.toString()}' '${outfile.toString()}'" - ] - } else { - // no need to copy if infile is the same as outfile - [] - } - } """ -mkdir -p "\$(dirname '${yamlFile}')" -echo "Storing state as yaml" -echo '${yamlBlob}' > '${yamlFile}' -echo "Copying output files to destination folder" -${copyCommands.join("\n ")} -""" + mkdir -p "\$(dirname '${yamlFile}')" + echo "Storing state as yaml" + cat > '${yamlFile}' << HERE +${yamlBlob} +HERE + """ } @@ -1806,13 +1942,10 @@ def publishStatesByConfig(Map args) { .replaceAll('\\$\\{key\\}', key_) def yamlDir = java.nio.file.Paths.get(yamlFilename).getParent() - // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where + // the processed state is a list of [key, value] tuples, where // - key is a String // - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path) - // - inputPath is a List[Path] - // - outputFilename is a List[String] // - (key, value) are the tuples that will be saved to the state.yaml file - // - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml) def processedState = config.allArguments .findAll { it.direction == "output" } @@ -1829,7 +1962,7 @@ def publishStatesByConfig(Map args) { // in the state as-is, but is not something that needs // to be copied from the source path to the dest path if (par.type != "file") { - return [[key: plainName_, value: value, inputPath: [], outputFilename: []]] + return [[key: plainName_, value: value]] } // if the orig state does not contain this filename, // it's an optional argument for which the user specified @@ -1860,13 +1993,9 @@ def publishStatesByConfig(Map args) { if (yamlDir != null) { value_ = yamlDir.relativize(value_) } - def inputPath = val instanceof File ? val.toPath() : val - [value: value_, inputPath: inputPath, outputFilename: filename_ix] + return value_ } - def transposedOutputs = ["value", "inputPath", "outputFilename"].collectEntries{ key -> - [key, outputPerFile.collect{dic -> dic[key]}] - } - return [[key: plainName_] + transposedOutputs] + return [["key": plainName_, "value": outputPerFile]] } else { def value_ = java.nio.file.Paths.get(filename) // if id contains a slash @@ -1874,18 +2003,17 @@ def publishStatesByConfig(Map args) { value_ = yamlDir.relativize(value_) } def inputPath = value instanceof File ? value.toPath() : value - return [[key: plainName_, value: value_, inputPath: [inputPath], outputFilename: [filename]]] + return [["key": plainName_, value: value_]] } } + def updatedState_ = processedState.collectEntries{[it.key, it.value]} - def inputPaths = processedState.collectMany{it.inputPath} - def outputFilenames = processedState.collectMany{it.outputFilename} // convert state to yaml blob def yamlBlob_ = toTaggedYamlBlob([id: id_] + updatedState_) - [id_, yamlBlob_, yamlFilename, inputPaths, outputFilenames] + [id_, yamlBlob_, yamlFilename] } | publishStatesProc emit: input_ch @@ -2559,7 +2687,8 @@ def _debug(workflowArgs, debugKey) { def workflowFactory(Map args, Map defaultWfArgs, Map meta) { def workflowArgs = processWorkflowArgs(args, defaultWfArgs, meta) def key_ = workflowArgs["key"] - + def multipleArgs = meta.config.allArguments.findAll{ it.multiple }.collect{it.plainName} + workflow workflowInstance { take: input_ @@ -2716,12 +2845,36 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { } // TODO: move some of the _meta.join_id wrangling to the safeJoin() function. - def chInitialOutput = chArgsWithDefaults + def chInitialOutputMulti = chArgsWithDefaults | _debug(workflowArgs, "processed") // run workflow | innerWorkflowFactory(workflowArgs) - // check output tuple - | map { id_, output_ -> + def chInitialOutputList = chInitialOutputMulti instanceof List ? chInitialOutputMulti : [chInitialOutputMulti] + assert chInitialOutputList.size() > 0: "should have emitted at least one output channel" + // Add a channel ID to the events, which designates the channel the event was emitted from as a running number + // This number is used to sort the events later when the events are gathered from across the channels. + def chInitialOutputListWithIndexedEvents = chInitialOutputList.withIndex().collect{channel, channelIndex -> + def newChannel = channel + | map {tuple -> + assert tuple instanceof List : + "Error in module '${key_}': element in output channel should be a tuple [id, data, ...otherargs...]\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Expected class: List. Found: tuple.getClass() is ${tuple.getClass()}" + + def newEvent = [channelIndex] + tuple + return newEvent + } + return newChannel + } + // Put the events into 1 channel, cover case where there is only one channel is emitted + def chInitialOutput = chInitialOutputList.size() > 1 ? \ + chInitialOutputListWithIndexedEvents[0].mix(*chInitialOutputListWithIndexedEvents.tail()) : \ + chInitialOutputListWithIndexedEvents[0] + def chInitialOutputProcessed = chInitialOutput + | map { tuple -> + def channelId = tuple[0] + def id_ = tuple[1] + def output_ = tuple[2] // see if output map contains metadata def meta_ = @@ -2734,19 +2887,94 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { output_ = output_.findAll{k, v -> k != "_meta"} // check value types - output_ = _processOutputValues(output_, meta.config, id_, key_) + output_ = _checkValidOutputArgument(output_, meta.config, id_, key_) - // simplify output if need be - if (workflowArgs.auto.simplifyOutput && output_.size() == 1) { - output_ = output_.values()[0] - } - - [join_id, id_, output_] + [join_id, channelId, id_, output_] } // | view{"chInitialOutput: ${it.take(3)}"} + // join the output [prev_id, channel_id, new_id, output] with the previous state [prev_id, state, ...] + def chPublishWithPreviousState = safeJoin(chInitialOutputProcessed, chRunFiltered, key_) + // input tuple format: [join_id, channel_id, id, output, prev_state, ...] + // output tuple format: [join_id, channel_id, id, new_state, ...] + | map{ tup -> + def new_state = workflowArgs.toState(tup.drop(2).take(3)) + tup.take(3) + [new_state] + tup.drop(5) + } + if (workflowArgs.auto.publish == "state") { + def chPublishFiles = chPublishWithPreviousState + // input tuple format: [join_id, channel_id, id, new_state, ...] + // output tuple format: [join_id, channel_id, id, new_state] + | map{ tup -> + tup.take(4) + } + + safeJoin(chPublishFiles, chArgsWithDefaults, key_) + // input tuple format: [join_id, channel_id, id, new_state, orig_state, ...] + // output tuple format: [id, new_state, orig_state] + | map { tup -> + tup.drop(2).take(3) + } + | publishFilesByConfig(key: key_, config: meta.config) + } + // Join the state from the events that were emitted from different channels + def chJoined = chInitialOutputProcessed + | map {tuple -> + def join_id = tuple[0] + def channel_id = tuple[1] + def id = tuple[2] + def other = tuple.drop(3) + // Below, groupTuple is used to join the events. To make sure resuming a workflow + // keeps working, the output state must be deterministic. This means the state needs to be + // sorted with groupTuple's has a 'sort' argument. This argument can be set to 'hash', + // but hashing the state when it is large can be problematic in terms of performance. + // Therefore, a custom comparator function is provided. We add the channel ID to the + // states so that we can use the channel ID to sort the items. + def stateWithChannelID = [[channel_id] * other.size(), other].transpose() + // A comparator that is provided to groupTuple's 'sort' argument is applied + // to all elements of the event tuple (that is not the 'id'). The comparator + // closure that is used below expects the input to be List. So the join_id and + // channel_id must also be wrapped in a list. + [[join_id], [channel_id], id] + stateWithChannelID + } + | groupTuple(by: 2, sort: {a, b -> a[0] <=> b[0]}, size: chInitialOutputList.size(), remainder: true) + | map {join_ids, _, id, statesWithChannelID -> + // Remove the channel IDs from the states + def states = statesWithChannelID.collect{it[1]} + def newJoinId = join_ids.flatten().unique{a, b -> a <=> b} + assert newJoinId.size() == 1: "Multiple events were emitted for '$id'." + def newJoinIdUnique = newJoinId[0] + + // Merge the states from the different channels + def newState = states.inject([:]){ old_state, state_to_add -> + return old_state + state_to_add.collectEntries{k, v -> + if (!multipleArgs.contains(k)) { + // if the key is not a multiple argument, we expect only one value + if (old_state.containsKey(k)) { + assert old_state[k] == v : "ID $id: multiple entries for argument $k were emitted." + } + [k, v] + } else { + // if the key is a multiple argument, append the different values into one list + def prevValue = old_state.getOrDefault(k, []) + def prevValueAsList = prevValue instanceof List ? prevValue : [prevValue] + [k, prevValueAsList + v] + } + } + } + + _checkAllRequiredOuputsPresent(newState, meta.config, id, key_) + + // simplify output if need be + if (workflowArgs.auto.simplifyOutput && newState.size() == 1) { + newState = newState.values()[0] + } + + return [newJoinIdUnique, id, newState] + } + // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_) + def chNewState = safeJoin(chJoined, chRunFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -2755,23 +2983,21 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { } if (workflowArgs.auto.publish == "state") { - def chPublish = chNewState + def chPublishStates = chNewState // input tuple format: [join_id, id, new_state, ...] // output tuple format: [join_id, id, new_state] | map{ tup -> tup.take(3) } - safeJoin(chPublish, chArgsWithDefaults, key_) + safeJoin(chPublishStates, chArgsWithDefaults, key_) // input tuple format: [join_id, id, new_state, orig_state, ...] // output tuple format: [id, new_state, orig_state] | map { tup -> tup.drop(1).take(3) - } + } | publishStatesByConfig(key: key_, config: meta.config) } - - // remove join_id and meta chReturn = chNewState | map { tup -> // input tuple format: [join_id, id, new_state, ...] @@ -2805,7 +3031,7 @@ meta = [ "resources_dir": moduleDir.toRealPath().normalize(), "config": processConfig(readJsonBlob('''{ "name" : "runner", - "version" : "v0.3.8", + "version" : "v0.3.9", "argument_groups" : [ { "name" : "Input arguments", @@ -2929,6 +3155,10 @@ meta = [ ], "description" : "Runner for demultiplexing of raw sequencing data", "status" : "enabled", + "scope" : { + "image" : "public", + "target" : "public" + }, "requirements" : { "commands" : [ "ps" @@ -3039,14 +3269,14 @@ meta = [ "runner" : "nextflow", "engine" : "native|native", "output" : "target/nextflow/runner", - "viash_version" : "0.9.0", - "git_commit" : "dafe2d829079bd5a9f5eb4e7cf63e92edbf1a742", + "viash_version" : "0.9.4", + "git_commit" : "820318c378d8119e2aa98768282e43c2aa017ba7", "git_remote" : "https://github.com/viash-hub/demultiplex", - "git_tag" : "v0.3.5-9-gdafe2d8" + "git_tag" : "v0.3.8-5-g820318c" }, "package_config" : { "name" : "demultiplex", - "version" : "v0.3.8", + "version" : "v0.3.9", "description" : "Demultiplexing pipeline\n", "info" : { "test_resources" : [ @@ -3056,14 +3286,14 @@ meta = [ } ] }, - "viash_version" : "0.9.0", + "viash_version" : "0.9.4", "source" : "src", "target" : "target", "config_mods" : [ - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n", + ".requirements.commands += ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v0.3.8'" + ".engines[.type == 'docker'].target_tag := 'v0.3.9'" ], "keywords" : [ "bioinformatics", diff --git a/target/nextflow/runner/nextflow.config b/target/nextflow/runner/nextflow.config index 9bdb0c0..d780db4 100644 --- a/target/nextflow/runner/nextflow.config +++ b/target/nextflow/runner/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'runner' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v0.3.8' + version = 'v0.3.9' description = 'Runner for demultiplexing of raw sequencing data' } diff --git a/target/nextflow/runner/nextflow_schema.json b/target/nextflow/runner/nextflow_schema.json index a7bb138..5c79973 100644 --- a/target/nextflow/runner/nextflow_schema.json +++ b/target/nextflow/runner/nextflow_schema.json @@ -80,10 +80,10 @@ "fastq_output": { "type": "string", - "description": "Type: `file`, default: `$id.$key.fastq_output.fastq_output`. ", - "help_text": "Type: `file`, default: `$id.$key.fastq_output.fastq_output`. " + "description": "Type: `file`, default: `fastq`. ", + "help_text": "Type: `file`, default: `fastq`. " , - "default":"$id.$key.fastq_output.fastq_output" + "default":"fastq" } @@ -91,10 +91,10 @@ "falco_output": { "type": "string", - "description": "Type: `file`, default: `$id.$key.falco_output.falco_output`. ", - "help_text": "Type: `file`, default: `$id.$key.falco_output.falco_output`. " + "description": "Type: `file`, default: `qc/fastqc`. ", + "help_text": "Type: `file`, default: `qc/fastqc`. " , - "default":"$id.$key.falco_output.falco_output" + "default":"qc/fastqc" } @@ -102,10 +102,10 @@ "multiqc_output": { "type": "string", - "description": "Type: `file`, default: `$id.$key.multiqc_output.html`. ", - "help_text": "Type: `file`, default: `$id.$key.multiqc_output.html`. " + "description": "Type: `file`, default: `qc/multiqc_report.html`. ", + "help_text": "Type: `file`, default: `qc/multiqc_report.html`. " , - "default":"$id.$key.multiqc_output.html" + "default":"qc/multiqc_report.html" }