Build branch v0.3 with version v0.3.9 (820318c)

Build pipeline: viash-hub.demultiplex.v0.3-5lwbd

Source commit: 820318c378

Source message: Bump version to v0.3.9

CHANGELOG.md

@@ -1,3 +1,21 @@
+# demultiplex v0.3.9
+
+## Bug fixes
+
+* Fix defaults for output arguments in nextflow schema's.
+
+* Fix an issue where an integer being passed to a argument with `type: double` resulted in an error (PR #44).
+
+## Minor changes
+
+* Bump viash to 0.9.4, which adds support for nextflow versions starting major version 25.01 (PR #43 and #44).
+
+# demultiplex v0.3.8
+
+## Bug fixes
+
+* Provide a proper error when a FASTQ file is empty after demultiplexing (PR #40).
+
 # demultiplex v0.3.7
 
 ## Minor updates

README.md

@@ -9,7 +9,7 @@ License](https://img.shields.io/github/license/viash-hub/demultiplex.svg)](https
 [![GitHub
 Issues](https://img.shields.io/github/issues/viash-hub/demultiplex.svg)](https://github.com/viash-hub/demultiplex/issues)
 [![Viash
-version](https://img.shields.io/badge/Viash-v0.9.1-blue)](https://viash.io)
+version](https://img.shields.io/badge/Viash-v0.9.4-blue)](https://viash.io)
 
 ## Workflow Overview
 The workflow executes the following steps: 
@@ -53,7 +53,7 @@ You can check if everything is working by getting the `--help` for a workflow:
 ```bash
 nextflow run \
 vsh/demultiplex \
--r v0.3.4 \
+-r v0.3.9 \
 --help
 ```
 
@@ -84,7 +84,7 @@ When starting nextflow using the CLI, you can use `-c` to provide the file to ne
 
 ```bash
 nextflow run vsh/demultiplex \
--r v0.3.4 \
+-r v0.3.9 \
 -main-script target/nextflow/runner/main.nf \
 --input "gs://viash-hub-test-data/demultiplex/v3/demultiplex_htrnaseq_meta/SingleCell-RNA_P3_2"  \
 --demultiplexer bclconvert \

_viash.yaml

@@ -1,5 +1,5 @@
 name: demultiplex
-version: v0.3.7
+version: v0.3.9
 description: |
   Demultiplexing pipeline
 license: MIT
@@ -12,10 +12,10 @@ info:
     - path: gs://viash-hub-test-data/demultiplex/v2/
       dest: testData
 
-viash_version: 0.9.0
+viash_version: 0.9.4
 
 config_mods: |
-  .requirements.commands := ['ps']
+  .requirements.commands += ['ps']
   .runners[.type == 'nextflow'].directives.tag := '$id'
   .resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}
   .runners[.type == 'nextflow'].config.script := 'includeConfig("nextflow_labels.config")'

src/dataflow/gather_fastqs_and_validate/main.nf

@@ -1,3 +1,26 @@
+import java.util.zip.GZIPInputStream
+import java.nio.file.Files
+import java.io.BufferedInputStream
+
+def is_empty(file_to_check){
+  /* 
+  Checks if a file has content
+  */
+  if (file_to_check.size() == 0) {
+    return true
+  }
+  def input_stream = Files.newInputStream(file_to_check)
+  def gzInputStream
+  try {
+    gzInputStream = new GZIPInputStream(new BufferedInputStream(input_stream))
+  } catch (java.io.EOFException ex) {
+    // This is not a gzipfile...
+    return false
+  }
+  def read_one_byte = gzInputStream.read()
+  return read_one_byte == -1
+}
+
 workflow run_wf {
   take:
     input_ch
@@ -78,6 +101,9 @@ workflow run_wf {
             "Found forward: ${forward_fastq} and reverse: ${reverse_fastq}."
           println "Found ${forward_fastq.size()} forward and ${reverse_fastq.size()} reverse " +
             "fastq files for sample ${sample_id}"
+
+          assert forward_fastq.every{!is_empty(it)} && reverse_fastq.every{!is_empty(it)}:
+            "A fastq file for sample '${sample_id}' appears to be empty!"
           def fastqs_state = [
             "fastq_forward": forward_fastq,
             "fastq_reverse": reverse_fastq,

target/executable/io/interop_summary_to_csv/.config.vsh.yaml

@@ -1,6 +1,6 @@
 name: "interop_summary_to_csv"
 namespace: "io"
-version: "v0.3.7"
+version: "v0.3.9"
 argument_groups:
 - name: "Input arguments"
   arguments:
@@ -43,8 +43,13 @@ resources:
   dest: "nextflow_labels.config"
 info: null
 status: "enabled"
+scope:
+  image: "public"
+  target: "public"
 requirements:
   commands:
+  - "summary"
+  - "index-summary"
   - "ps"
 license: "MIT"
 links:
@@ -121,7 +126,7 @@ engines:
   id: "docker"
   image: "debian:stable-slim"
   target_registry: "images.viash-hub.com"
-  target_tag: "v0.3.7"
+  target_tag: "v0.3.9"
   namespace_separator: "/"
   setup:
   - type: "apt"
@@ -145,29 +150,29 @@ build_info:
   engine: "docker|native"
   output: "target/executable/io/interop_summary_to_csv"
   executable: "target/executable/io/interop_summary_to_csv/interop_summary_to_csv"
-  viash_version: "0.9.0"
+  viash_version: "0.9.4"
-  git_commit: "e177226c52ca1dd5a5cf9bb0bfc108812e2aa332"
+  git_commit: "820318c378d8119e2aa98768282e43c2aa017ba7"
   git_remote: "https://github.com/viash-hub/demultiplex"
-  git_tag: "v0.3.5-6-ge177226"
+  git_tag: "v0.3.8-5-g820318c"
 package_config:
   name: "demultiplex"
-  version: "v0.3.7"
+  version: "v0.3.9"
   description: "Demultiplexing pipeline\n"
   info:
     test_resources:
     - path: "gs://viash-hub-test-data/demultiplex/v2/"
       dest: "testData"
-  viash_version: "0.9.0"
+  viash_version: "0.9.4"
   source: "src"
   target: "target"
   config_mods:
-  - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\
+  - ".requirements.commands += ['ps']\n.runners[.type == 'nextflow'].directives.tag\
     \ := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n\
     .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\
     )'\n"
   - ".engines += { type: \"native\" }"
   - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
-  - ".engines[.type == 'docker'].target_tag := 'v0.3.7'"
+  - ".engines[.type == 'docker'].target_tag := 'v0.3.9'"
   keywords:
   - "bioinformatics"
   - "sequence"

target/executable/io/interop_summary_to_csv/interop_summary_to_csv

@@ -1,8 +1,8 @@
 #!/usr/bin/env bash
 
-# interop_summary_to_csv v0.3.7
+# interop_summary_to_csv v0.3.9
 # 
-# This wrapper script is auto-generated by viash 0.9.0 and is thus a derivative
+# This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative
 # work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data
 # Intuitive.
 # 
@@ -169,22 +169,6 @@ VIASH_META_CONFIG="$VIASH_META_RESOURCES_DIR/.config.vsh.yaml"
 VIASH_META_TEMP_DIR="$VIASH_TEMP"
 
 
-# ViashHelp: Display helpful explanation about this executable
-function ViashHelp {
-  echo "interop_summary_to_csv v0.3.7"
-  echo ""
-  echo "Input arguments:"
-  echo "    --input"
-  echo "        type: file, required parameter, file must exist"
-  echo "        Sequencing run folder (*not* InterOp folder)."
-  echo ""
-  echo "Output arguments:"
-  echo "    --output_run_summary"
-  echo "        type: file, required parameter, output, file must exist"
-  echo ""
-  echo "    --output_index_summary"
-  echo "        type: file, required parameter, output, file must exist"
-}
 
 # initialise variables
 VIASH_MODE='run'
@@ -470,10 +454,10 @@ tar -C /tmp/ --no-same-owner --no-same-permissions -xvf /tmp/interop.tar.gz && \
 mv /tmp/interop-1.3.1-Linux-GNU/bin/index-summary /tmp/interop-1.3.1-Linux-GNU/bin/summary /usr/local/bin/
 
 LABEL org.opencontainers.image.description="Companion container for running component io interop_summary_to_csv"
-LABEL org.opencontainers.image.created="2025-03-20T20:32:33Z"
+LABEL org.opencontainers.image.created="2025-04-25T12:13:39Z"
 LABEL org.opencontainers.image.source="https://github.com/viash-hub/demultiplex"
-LABEL org.opencontainers.image.revision="e177226c52ca1dd5a5cf9bb0bfc108812e2aa332"
+LABEL org.opencontainers.image.revision="820318c378d8119e2aa98768282e43c2aa017ba7"
-LABEL org.opencontainers.image.version="v0.3.7"
+LABEL org.opencontainers.image.version="v0.3.9"
 
 VIASHDOCKER
   fi
@@ -587,6 +571,48 @@ fi
 # initialise docker variables
 VIASH_DOCKER_RUN_ARGS=(-i --rm)
 
+
+# ViashHelp: Display helpful explanation about this executable
+function ViashHelp {
+  echo "interop_summary_to_csv v0.3.9"
+  echo ""
+  echo "Input arguments:"
+  echo "    --input"
+  echo "        type: file, required parameter, file must exist"
+  echo "        Sequencing run folder (*not* InterOp folder)."
+  echo ""
+  echo "Output arguments:"
+  echo "    --output_run_summary"
+  echo "        type: file, required parameter, output, file must exist"
+  echo ""
+  echo "    --output_index_summary"
+  echo "        type: file, required parameter, output, file must exist"
+  echo ""
+  echo "Viash built in Computational Requirements:"
+  echo "    ---cpus=INT"
+  echo "        Number of CPUs to use"
+  echo "    ---memory=STRING"
+  echo "        Amount of memory to use. Examples: 4GB, 3MiB."
+  echo ""
+  echo "Viash built in Docker:"
+  echo "    ---setup=STRATEGY"
+  echo "        Setup the docker container. Options are: alwaysbuild, alwayscachedbuild, ifneedbebuild, ifneedbecachedbuild, alwayspull, alwayspullelsebuild, alwayspullelsecachedbuild, ifneedbepull, ifneedbepullelsebuild, ifneedbepullelsecachedbuild, push, pushifnotpresent, donothing."
+  echo "        Default: ifneedbepullelsecachedbuild"
+  echo "    ---dockerfile"
+  echo "        Print the dockerfile to stdout."
+  echo "    ---docker_run_args=ARG"
+  echo "        Provide runtime arguments to Docker. See the documentation on \`docker run\` for more information."
+  echo "    ---docker_image_id"
+  echo "        Print the docker image id to stdout."
+  echo "    ---debug"
+  echo "        Enter the docker container for debugging purposes."
+  echo ""
+  echo "Viash built in Engines:"
+  echo "    ---engine=ENGINE_ID"
+  echo "        Specify the engine to use. Options are: docker, native."
+  echo "        Default: docker"
+}
+
 # initialise array
 VIASH_POSITIONAL_ARGS=''
 
@@ -609,7 +635,7 @@ while [[ $# -gt 0 ]]; do
             shift 1
             ;;
         --version)
-            echo "interop_summary_to_csv v0.3.7"
+            echo "interop_summary_to_csv v0.3.9"
             exit
             ;;
         --input)
@@ -733,7 +759,7 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then
 
   # determine docker image id
   if [[ "$VIASH_ENGINE_ID" == 'docker' ]]; then
-    VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/demultiplex/io/interop_summary_to_csv:v0.3.7'
+    VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/demultiplex/io/interop_summary_to_csv:v0.3.9'
   fi
 
   # print dockerfile
@@ -755,13 +781,13 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then
   # build docker image
   elif [ "$VIASH_MODE" == "setup" ]; then
     ViashDockerSetup "$VIASH_DOCKER_IMAGE_ID" "$VIASH_SETUP_STRATEGY"
-    ViashDockerCheckCommands "$VIASH_DOCKER_IMAGE_ID" 'ps' 'bash'
+    ViashDockerCheckCommands "$VIASH_DOCKER_IMAGE_ID" 'summary' 'index-summary' 'ps' 'bash'
     exit 0
   fi
 
   # check if docker image exists
   ViashDockerSetup "$VIASH_DOCKER_IMAGE_ID" ifneedbepullelsecachedbuild
-  ViashDockerCheckCommands "$VIASH_DOCKER_IMAGE_ID" 'ps' 'bash'
+  ViashDockerCheckCommands "$VIASH_DOCKER_IMAGE_ID" 'summary' 'index-summary' 'ps' 'bash'
 fi
 
 # setting computational defaults

target/executable/io/publish/.config.vsh.yaml

@@ -1,6 +1,6 @@
 name: "publish"
 namespace: "io"
-version: "v0.3.7"
+version: "v0.3.9"
 argument_groups:
 - name: "Input arguments"
   arguments:
@@ -100,6 +100,9 @@ resources:
 description: "Publish the processed results of the run"
 info: null
 status: "enabled"
+scope:
+  image: "public"
+  target: "public"
 requirements:
   commands:
   - "ps"
@@ -178,7 +181,7 @@ engines:
   id: "docker"
   image: "debian:stable-slim"
   target_registry: "images.viash-hub.com"
-  target_tag: "v0.3.7"
+  target_tag: "v0.3.9"
   namespace_separator: "/"
   setup:
   - type: "apt"
@@ -195,29 +198,29 @@ build_info:
   engine: "docker|native"
   output: "target/executable/io/publish"
   executable: "target/executable/io/publish/publish"
-  viash_version: "0.9.0"
+  viash_version: "0.9.4"
-  git_commit: "e177226c52ca1dd5a5cf9bb0bfc108812e2aa332"
+  git_commit: "820318c378d8119e2aa98768282e43c2aa017ba7"
   git_remote: "https://github.com/viash-hub/demultiplex"
-  git_tag: "v0.3.5-6-ge177226"
+  git_tag: "v0.3.8-5-g820318c"
 package_config:
   name: "demultiplex"
-  version: "v0.3.7"
+  version: "v0.3.9"
   description: "Demultiplexing pipeline\n"
   info:
     test_resources:
     - path: "gs://viash-hub-test-data/demultiplex/v2/"
       dest: "testData"
-  viash_version: "0.9.0"
+  viash_version: "0.9.4"
   source: "src"
   target: "target"
   config_mods:
-  - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\
+  - ".requirements.commands += ['ps']\n.runners[.type == 'nextflow'].directives.tag\
     \ := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n\
     .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\
     )'\n"
   - ".engines += { type: \"native\" }"
   - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
-  - ".engines[.type == 'docker'].target_tag := 'v0.3.7'"
+  - ".engines[.type == 'docker'].target_tag := 'v0.3.9'"
   keywords:
   - "bioinformatics"
   - "sequence"

target/executable/io/publish/publish

@@ -1,8 +1,8 @@
 #!/usr/bin/env bash
 
-# publish v0.3.7
+# publish v0.3.9
 # 
-# This wrapper script is auto-generated by viash 0.9.0 and is thus a derivative
+# This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative
 # work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data
 # Intuitive.
 # 
@@ -169,46 +169,6 @@ VIASH_META_CONFIG="$VIASH_META_RESOURCES_DIR/.config.vsh.yaml"
 VIASH_META_TEMP_DIR="$VIASH_TEMP"
 
 
-# ViashHelp: Display helpful explanation about this executable
-function ViashHelp {
-  echo "publish v0.3.7"
-  echo ""
-  echo "Publish the processed results of the run"
-  echo ""
-  echo "Input arguments:"
-  echo "    --input"
-  echo "        type: file, required parameter, file must exist"
-  echo "        Directory to write fastq data to"
-  echo ""
-  echo "    --input_falco"
-  echo "        type: file, required parameter, multiple values allowed, file must exist"
-  echo "        Directory to write falco output to"
-  echo ""
-  echo "    --input_multiqc"
-  echo "        type: file, required parameter, file must exist"
-  echo "        Location where to write the MultiQC report to."
-  echo ""
-  echo "    --input_run_information"
-  echo "        type: file, required parameter, file must exist"
-  echo "        Location where to write the run information to."
-  echo ""
-  echo "Output arguments:"
-  echo "    --output"
-  echo "        type: file, output, file must exist"
-  echo "        default: fastq"
-  echo ""
-  echo "    --output_falco"
-  echo "        type: file, output, file must exist"
-  echo "        default: qc/fastqc"
-  echo ""
-  echo "    --output_multiqc"
-  echo "        type: file, output, file must exist"
-  echo "        default: qc/multiqc_report.html"
-  echo ""
-  echo "    --output_run_information"
-  echo "        type: file, output, file must exist"
-  echo "        default: run_information.csv"
-}
 
 # initialise variables
 VIASH_MODE='run'
@@ -490,10 +450,10 @@ RUN apt-get update && \
   rm -rf /var/lib/apt/lists/*
 
 LABEL org.opencontainers.image.description="Companion container for running component io publish"
-LABEL org.opencontainers.image.created="2025-03-20T20:32:33Z"
+LABEL org.opencontainers.image.created="2025-04-25T12:13:38Z"
 LABEL org.opencontainers.image.source="https://github.com/viash-hub/demultiplex"
-LABEL org.opencontainers.image.revision="e177226c52ca1dd5a5cf9bb0bfc108812e2aa332"
+LABEL org.opencontainers.image.revision="820318c378d8119e2aa98768282e43c2aa017ba7"
-LABEL org.opencontainers.image.version="v0.3.7"
+LABEL org.opencontainers.image.version="v0.3.9"
 
 VIASHDOCKER
   fi
@@ -607,6 +567,72 @@ fi
 # initialise docker variables
 VIASH_DOCKER_RUN_ARGS=(-i --rm)
 
+
+# ViashHelp: Display helpful explanation about this executable
+function ViashHelp {
+  echo "publish v0.3.9"
+  echo ""
+  echo "Publish the processed results of the run"
+  echo ""
+  echo "Input arguments:"
+  echo "    --input"
+  echo "        type: file, required parameter, file must exist"
+  echo "        Directory to write fastq data to"
+  echo ""
+  echo "    --input_falco"
+  echo "        type: file, required parameter, multiple values allowed, file must exist"
+  echo "        Directory to write falco output to"
+  echo ""
+  echo "    --input_multiqc"
+  echo "        type: file, required parameter, file must exist"
+  echo "        Location where to write the MultiQC report to."
+  echo ""
+  echo "    --input_run_information"
+  echo "        type: file, required parameter, file must exist"
+  echo "        Location where to write the run information to."
+  echo ""
+  echo "Output arguments:"
+  echo "    --output"
+  echo "        type: file, output, file must exist"
+  echo "        default: fastq"
+  echo ""
+  echo "    --output_falco"
+  echo "        type: file, output, file must exist"
+  echo "        default: qc/fastqc"
+  echo ""
+  echo "    --output_multiqc"
+  echo "        type: file, output, file must exist"
+  echo "        default: qc/multiqc_report.html"
+  echo ""
+  echo "    --output_run_information"
+  echo "        type: file, output, file must exist"
+  echo "        default: run_information.csv"
+  echo ""
+  echo "Viash built in Computational Requirements:"
+  echo "    ---cpus=INT"
+  echo "        Number of CPUs to use"
+  echo "    ---memory=STRING"
+  echo "        Amount of memory to use. Examples: 4GB, 3MiB."
+  echo ""
+  echo "Viash built in Docker:"
+  echo "    ---setup=STRATEGY"
+  echo "        Setup the docker container. Options are: alwaysbuild, alwayscachedbuild, ifneedbebuild, ifneedbecachedbuild, alwayspull, alwayspullelsebuild, alwayspullelsecachedbuild, ifneedbepull, ifneedbepullelsebuild, ifneedbepullelsecachedbuild, push, pushifnotpresent, donothing."
+  echo "        Default: ifneedbepullelsecachedbuild"
+  echo "    ---dockerfile"
+  echo "        Print the dockerfile to stdout."
+  echo "    ---docker_run_args=ARG"
+  echo "        Provide runtime arguments to Docker. See the documentation on \`docker run\` for more information."
+  echo "    ---docker_image_id"
+  echo "        Print the docker image id to stdout."
+  echo "    ---debug"
+  echo "        Enter the docker container for debugging purposes."
+  echo ""
+  echo "Viash built in Engines:"
+  echo "    ---engine=ENGINE_ID"
+  echo "        Specify the engine to use. Options are: docker, native."
+  echo "        Default: docker"
+}
+
 # initialise array
 VIASH_POSITIONAL_ARGS=''
 
@@ -629,7 +655,7 @@ while [[ $# -gt 0 ]]; do
             shift 1
             ;;
         --version)
-            echo "publish v0.3.7"
+            echo "publish v0.3.9"
             exit
             ;;
         --input)
@@ -814,7 +840,7 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then
 
   # determine docker image id
   if [[ "$VIASH_ENGINE_ID" == 'docker' ]]; then
-    VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/demultiplex/io/publish:v0.3.7'
+    VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/demultiplex/io/publish:v0.3.9'
   fi
 
   # print dockerfile

target/executable/io/untar/.config.vsh.yaml

@@ -1,6 +1,6 @@
 name: "untar"
 namespace: "io"
-version: "v0.3.7"
+version: "v0.3.9"
 argument_groups:
 - name: "Input arguments"
   arguments:
@@ -57,6 +57,9 @@ test_resources:
   is_executable: true
 info: null
 status: "enabled"
+scope:
+  image: "public"
+  target: "public"
 requirements:
   commands:
   - "ps"
@@ -135,7 +138,7 @@ engines:
   id: "docker"
   image: "debian:stable-slim"
   target_registry: "images.viash-hub.com"
-  target_tag: "v0.3.7"
+  target_tag: "v0.3.9"
   namespace_separator: "/"
   setup:
   - type: "apt"
@@ -152,29 +155,29 @@ build_info:
   engine: "docker|native"
   output: "target/executable/io/untar"
   executable: "target/executable/io/untar/untar"
-  viash_version: "0.9.0"
+  viash_version: "0.9.4"
-  git_commit: "e177226c52ca1dd5a5cf9bb0bfc108812e2aa332"
+  git_commit: "820318c378d8119e2aa98768282e43c2aa017ba7"
   git_remote: "https://github.com/viash-hub/demultiplex"
-  git_tag: "v0.3.5-6-ge177226"
+  git_tag: "v0.3.8-5-g820318c"
 package_config:
   name: "demultiplex"
-  version: "v0.3.7"
+  version: "v0.3.9"
   description: "Demultiplexing pipeline\n"
   info:
     test_resources:
     - path: "gs://viash-hub-test-data/demultiplex/v2/"
       dest: "testData"
-  viash_version: "0.9.0"
+  viash_version: "0.9.4"
   source: "src"
   target: "target"
   config_mods:
-  - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\
+  - ".requirements.commands += ['ps']\n.runners[.type == 'nextflow'].directives.tag\
     \ := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n\
     .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\
     )'\n"
   - ".engines += { type: \"native\" }"
   - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
-  - ".engines[.type == 'docker'].target_tag := 'v0.3.7'"
+  - ".engines[.type == 'docker'].target_tag := 'v0.3.9'"
   keywords:
   - "bioinformatics"
   - "sequence"

target/executable/io/untar/untar

@@ -1,8 +1,8 @@
 #!/usr/bin/env bash
 
-# untar v0.3.7
+# untar v0.3.9
 # 
-# This wrapper script is auto-generated by viash 0.9.0 and is thus a derivative
+# This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative
 # work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data
 # Intuitive.
 # 
@@ -169,32 +169,6 @@ VIASH_META_CONFIG="$VIASH_META_RESOURCES_DIR/.config.vsh.yaml"
 VIASH_META_TEMP_DIR="$VIASH_TEMP"
 
 
-# ViashHelp: Display helpful explanation about this executable
-function ViashHelp {
-  echo "untar v0.3.7"
-  echo ""
-  echo "Unpack a .tar file. When the contents of the .tar file is just a single"
-  echo "directory,"
-  echo "put the contents of the directory into the output folder instead of that"
-  echo "directory."
-  echo ""
-  echo "Input arguments:"
-  echo "    --input"
-  echo "        type: file, required parameter, file must exist"
-  echo "        Tarball file to be unpacked."
-  echo ""
-  echo "Output arguments:"
-  echo "    --output"
-  echo "        type: file, required parameter, output, file must exist"
-  echo "        Directory to write the contents of the .tar file to."
-  echo ""
-  echo "Other arguments:"
-  echo "    -e, --exclude"
-  echo "        type: string"
-  echo "        example: docs/figures"
-  echo "        Prevents any file or member whose name matches the shell wildcard"
-  echo "        (pattern) from being extracted."
-}
 
 # initialise variables
 VIASH_MODE='run'
@@ -476,10 +450,10 @@ RUN apt-get update && \
   rm -rf /var/lib/apt/lists/*
 
 LABEL org.opencontainers.image.description="Companion container for running component io untar"
-LABEL org.opencontainers.image.created="2025-03-20T20:32:32Z"
+LABEL org.opencontainers.image.created="2025-04-25T12:13:38Z"
 LABEL org.opencontainers.image.source="https://github.com/viash-hub/demultiplex"
-LABEL org.opencontainers.image.revision="e177226c52ca1dd5a5cf9bb0bfc108812e2aa332"
+LABEL org.opencontainers.image.revision="820318c378d8119e2aa98768282e43c2aa017ba7"
-LABEL org.opencontainers.image.version="v0.3.7"
+LABEL org.opencontainers.image.version="v0.3.9"
 
 VIASHDOCKER
   fi
@@ -593,6 +567,58 @@ fi
 # initialise docker variables
 VIASH_DOCKER_RUN_ARGS=(-i --rm)
 
+
+# ViashHelp: Display helpful explanation about this executable
+function ViashHelp {
+  echo "untar v0.3.9"
+  echo ""
+  echo "Unpack a .tar file. When the contents of the .tar file is just a single"
+  echo "directory,"
+  echo "put the contents of the directory into the output folder instead of that"
+  echo "directory."
+  echo ""
+  echo "Input arguments:"
+  echo "    --input"
+  echo "        type: file, required parameter, file must exist"
+  echo "        Tarball file to be unpacked."
+  echo ""
+  echo "Output arguments:"
+  echo "    --output"
+  echo "        type: file, required parameter, output, file must exist"
+  echo "        Directory to write the contents of the .tar file to."
+  echo ""
+  echo "Other arguments:"
+  echo "    -e, --exclude"
+  echo "        type: string"
+  echo "        example: docs/figures"
+  echo "        Prevents any file or member whose name matches the shell wildcard"
+  echo "        (pattern) from being extracted."
+  echo ""
+  echo "Viash built in Computational Requirements:"
+  echo "    ---cpus=INT"
+  echo "        Number of CPUs to use"
+  echo "    ---memory=STRING"
+  echo "        Amount of memory to use. Examples: 4GB, 3MiB."
+  echo ""
+  echo "Viash built in Docker:"
+  echo "    ---setup=STRATEGY"
+  echo "        Setup the docker container. Options are: alwaysbuild, alwayscachedbuild, ifneedbebuild, ifneedbecachedbuild, alwayspull, alwayspullelsebuild, alwayspullelsecachedbuild, ifneedbepull, ifneedbepullelsebuild, ifneedbepullelsecachedbuild, push, pushifnotpresent, donothing."
+  echo "        Default: ifneedbepullelsecachedbuild"
+  echo "    ---dockerfile"
+  echo "        Print the dockerfile to stdout."
+  echo "    ---docker_run_args=ARG"
+  echo "        Provide runtime arguments to Docker. See the documentation on \`docker run\` for more information."
+  echo "    ---docker_image_id"
+  echo "        Print the docker image id to stdout."
+  echo "    ---debug"
+  echo "        Enter the docker container for debugging purposes."
+  echo ""
+  echo "Viash built in Engines:"
+  echo "    ---engine=ENGINE_ID"
+  echo "        Specify the engine to use. Options are: docker, native."
+  echo "        Default: docker"
+}
+
 # initialise array
 VIASH_POSITIONAL_ARGS=''
 
@@ -615,7 +641,7 @@ while [[ $# -gt 0 ]]; do
             shift 1
             ;;
         --version)
-            echo "untar v0.3.7"
+            echo "untar v0.3.9"
             exit
             ;;
         --input)
@@ -745,7 +771,7 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then
 
   # determine docker image id
   if [[ "$VIASH_ENGINE_ID" == 'docker' ]]; then
-    VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/demultiplex/io/untar:v0.3.7'
+    VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/demultiplex/io/untar:v0.3.9'
   fi
 
   # print dockerfile

target/nextflow/dataflow/combine_samples/.config.vsh.yaml

@@ -1,6 +1,6 @@
 name: "combine_samples"
 namespace: "dataflow"
-version: "v0.3.7"
+version: "v0.3.9"
 argument_groups:
 - name: "Input arguments"
   arguments:
@@ -80,6 +80,9 @@ description: "Combine fastq files from across samples into one event with a list
   \ fastq files per orientation."
 info: null
 status: "enabled"
+scope:
+  image: "public"
+  target: "public"
 requirements:
   commands:
   - "ps"
@@ -161,29 +164,29 @@ build_info:
   engine: "native|native"
   output: "target/nextflow/dataflow/combine_samples"
   executable: "target/nextflow/dataflow/combine_samples/main.nf"
-  viash_version: "0.9.0"
+  viash_version: "0.9.4"
-  git_commit: "e177226c52ca1dd5a5cf9bb0bfc108812e2aa332"
+  git_commit: "820318c378d8119e2aa98768282e43c2aa017ba7"
   git_remote: "https://github.com/viash-hub/demultiplex"
-  git_tag: "v0.3.5-6-ge177226"
+  git_tag: "v0.3.8-5-g820318c"
 package_config:
   name: "demultiplex"
-  version: "v0.3.7"
+  version: "v0.3.9"
   description: "Demultiplexing pipeline\n"
   info:
     test_resources:
     - path: "gs://viash-hub-test-data/demultiplex/v2/"
       dest: "testData"
-  viash_version: "0.9.0"
+  viash_version: "0.9.4"
   source: "src"
   target: "target"
   config_mods:
-  - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\
+  - ".requirements.commands += ['ps']\n.runners[.type == 'nextflow'].directives.tag\
     \ := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n\
     .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\
     )'\n"
   - ".engines += { type: \"native\" }"
   - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
-  - ".engines[.type == 'docker'].target_tag := 'v0.3.7'"
+  - ".engines[.type == 'docker'].target_tag := 'v0.3.9'"
   keywords:
   - "bioinformatics"
   - "sequence"

target/nextflow/dataflow/combine_samples/main.nf

@@ -1,6 +1,6 @@
-// combine_samples v0.3.7
+// combine_samples v0.3.9
 // 
-// This wrapper script is auto-generated by viash 0.9.0 and is thus a derivative
+// This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative
 // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data
 // Intuitive.
 // 
@@ -82,64 +82,56 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi
       foundClass = "List[${e.foundClass}]"
     }
   } else if (par.type == "string") {
-    // cast to string if need be
+    // cast to string if need be. only cast if the value is a GString
     if (value instanceof GString) {
-      value = value.toString()
+      value = value as String
     }
     expectedClass = value instanceof String ? null : "String"
   } else if (par.type == "integer") {
     // cast to integer if need be
-    if (value instanceof String) {
+    if (value !instanceof Integer) {
       try {
-        value = value.toInteger()
+        value = value as Integer
       } catch (NumberFormatException e) {
-        // do nothing
+        expectedClass = "Integer"
       }
     }
-    if (value instanceof java.math.BigInteger) {
-      value = value.intValue()
-    }
-    expectedClass = value instanceof Integer ? null : "Integer"
   } else if (par.type == "long") {
     // cast to long if need be
-    if (value instanceof String) {
+    if (value !instanceof Long) {
       try {
-        value = value.toLong()
+        value = value as Long
       } catch (NumberFormatException e) {
-        // do nothing
+        expectedClass = "Long"
       }
     }
-    if (value instanceof Integer) {
-      value = value.toLong()
-    }
-    expectedClass = value instanceof Long ? null : "Long"
   } else if (par.type == "double") {
     // cast to double if need be
-    if (value instanceof String) {
+    if (value !instanceof Double) {
       try {
-        value = value.toDouble()
+        value = value as Double
       } catch (NumberFormatException e) {
-        // do nothing
+        expectedClass = "Double"
       }
     }
-    if (value instanceof java.math.BigDecimal) {
+  } else if (par.type == "float") {
-      value = value.doubleValue()
+    // cast to float if need be
+    if (value !instanceof Float) {
+      try {
+        value = value as Float
+      } catch (NumberFormatException e) {
+        expectedClass = "Float"
+      }
     }
-    if (value instanceof Float) {
-      value = value.toDouble()
-    }
-    expectedClass = value instanceof Double ? null : "Double"
   } else if (par.type == "boolean" | par.type == "boolean_true" | par.type == "boolean_false") {
     // cast to boolean if need be
-    if (value instanceof String) {
+    if (value !instanceof Boolean) {
-      def valueLower = value.toLowerCase()
+      try {
-      if (valueLower == "true") {
+        value = value as Boolean
-        value = true
+      } catch (Exception e) {
-      } else if (valueLower == "false") {
+        expectedClass = "Boolean"
-        value = false
       }
     }
-    expectedClass = value instanceof Boolean ? null : "Boolean"
   } else if (par.type == "file" && (par.direction == "input" || stage == "output")) {
     // cast to path if need be
     if (value instanceof String) {
@@ -151,10 +143,13 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi
     expectedClass = value instanceof Path ? null : "Path"
   } else if (par.type == "file" && stage == "input" && par.direction == "output") {
     // cast to string if need be
-    if (value instanceof GString) {
+    if (value !instanceof String) {
-      value = value.toString()
+      try {
+        value = value as String
+      } catch (Exception e) {
+        expectedClass = "String"
+      }
     }
-    expectedClass = value instanceof String ? null : "String"
   } else {
     // didn't find a match for par.type
     expectedClass = par.type
@@ -173,7 +168,7 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi
 Map _processInputValues(Map inputs, Map config, String id, String key) {
   if (!workflow.stubRun) {
     config.allArguments.each { arg ->
-      if (arg.required) {
+      if (arg.required && arg.direction == "input") {
         assert inputs.containsKey(arg.plainName) && inputs.get(arg.plainName) != null : 
           "Error in module '${key}' id '${id}': required input argument '${arg.plainName}' is missing"
       }
@@ -192,15 +187,8 @@ Map _processInputValues(Map inputs, Map config, String id, String key) {
 }
 
 // helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_processOutputValues.nf'
-Map _processOutputValues(Map outputs, Map config, String id, String key) {
+Map _checkValidOutputArgument(Map outputs, Map config, String id, String key) {
   if (!workflow.stubRun) {
-    config.allArguments.each { arg ->
-      if (arg.direction == "output" && arg.required) {
-        assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : 
-          "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing"
-      }
-    }
-
     outputs = outputs.collectEntries { name, value ->
       def par = config.allArguments.find { it.plainName == name && it.direction == "output" }
       assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid output argument"
@@ -213,6 +201,16 @@ Map _processOutputValues(Map outputs, Map config, String id, String key) {
   return outputs
 }
 
+void _checkAllRequiredOuputsPresent(Map outputs, Map config, String id, String key) {
+  if (!workflow.stubRun) {
+    config.allArguments.each { arg ->
+      if (arg.direction == "output" && arg.required) {
+        assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : 
+          "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing"
+      }
+    }
+  }
+}
 // helper file: 'src/main/resources/io/viash/runners/nextflow/channel/IDChecker.nf'
 class IDChecker {
   final def items = [] as Set
@@ -1666,6 +1664,162 @@ def joinStates(Closure apply_) {
   }
   return joinStatesWf
 }
+// helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishFiles.nf'
+def publishFiles(Map args) {
+  def key_ = args.get("key")
+
+  assert key_ != null : "publishFiles: key must be specified"
+  
+  workflow publishFilesWf {
+    take: input_ch
+    main:
+      input_ch
+        | map { tup ->
+          def id_ = tup[0]
+          def state_ = tup[1]
+
+          // the input files and the target output filenames
+          def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose()
+          def inputFiles_ = inputoutputFilenames_[0]
+          def outputFilenames_ = inputoutputFilenames_[1]
+
+          [id_, inputFiles_, outputFilenames_]
+        }
+        | publishFilesProc
+    emit: input_ch
+  }
+  return publishFilesWf
+}
+
+process publishFilesProc {
+  // todo: check publishpath?
+  publishDir path: "${getPublishDir()}/", mode: "copy"
+  tag "$id"
+  input:
+    tuple val(id), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles)
+  output:
+    tuple val(id), path{outputFiles}
+  script:
+  def copyCommands = [
+    inputFiles instanceof List ? inputFiles : [inputFiles],
+    outputFiles instanceof List ? outputFiles : [outputFiles]
+  ]
+    .transpose()
+    .collectMany{infile, outfile ->
+      if (infile.toString() != outfile.toString()) {
+        [
+          "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"",
+          "cp -r '${infile.toString()}' '${outfile.toString()}'"
+        ]
+      } else {
+        // no need to copy if infile is the same as outfile
+        []
+      }
+    }
+  """
+  echo "Copying output files to destination folder"
+  ${copyCommands.join("\n  ")}
+  """
+}
+
+
+// this assumes that the state contains no other values other than those specified in the config
+def publishFilesByConfig(Map args) {
+  def config = args.get("config")
+  assert config != null : "publishFilesByConfig: config must be specified"
+
+  def key_ = args.get("key", config.name)
+  assert key_ != null : "publishFilesByConfig: key must be specified"
+  
+  workflow publishFilesSimpleWf {
+    take: input_ch
+    main:
+      input_ch
+        | map { tup ->
+          def id_ = tup[0]
+          def state_ = tup[1] // e.g. [output: new File("myoutput.h5ad"), k: 10]
+          def origState_ = tup[2] // e.g. [output: '$id.$key.foo.h5ad']
+
+
+          // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where
+          //   - key is a String
+          //   - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path)
+          //   - inputPath is a List[Path]
+          //   - outputFilename is a List[String]
+          //   - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml)
+          def processedState =
+            config.allArguments
+              .findAll { it.direction == "output" }
+              .collectMany { par ->
+                def plainName_ = par.plainName
+                // if the state does not contain the key, it's an
+                // optional argument for which the component did 
+                // not generate any output OR multiple channels were emitted
+                // and the output was just not added to using the channel
+                // that is now being parsed
+                if (!state_.containsKey(plainName_)) {
+                  return []
+                }
+                def value = state_[plainName_]
+                // if the parameter is not a file, it should be stored
+                // in the state as-is, but is not something that needs 
+                // to be copied from the source path to the dest path
+                if (par.type != "file") {
+                  return [[inputPath: [], outputFilename: []]]
+                }
+                // if the orig state does not contain this filename,
+                // it's an optional argument for which the user specified
+                // that it should not be returned as a state
+                if (!origState_.containsKey(plainName_)) {
+                  return []
+                }
+                def filenameTemplate = origState_[plainName_]
+                // if the pararameter is multiple: true, fetch the template
+                if (par.multiple && filenameTemplate instanceof List) {
+                  filenameTemplate = filenameTemplate[0]
+                }
+                // instantiate the template
+                def filename = filenameTemplate
+                  .replaceAll('\\$id', id_)
+                  .replaceAll('\\$\\{id\\}', id_)
+                  .replaceAll('\\$key', key_)
+                  .replaceAll('\\$\\{key\\}', key_)
+                if (par.multiple) {
+                  // if the parameter is multiple: true, the filename
+                  // should contain a wildcard '*' that is replaced with
+                  // the index of the file
+                  assert filename.contains("*") : "Module '${key_}' id '${id_}': Multiple output files specified, but no wildcard '*' in the filename: ${filename}"
+                  def outputPerFile = value.withIndex().collect{ val, ix ->
+                    def filename_ix = filename.replace("*", ix.toString())
+                    def inputPath = val instanceof File ? val.toPath() : val
+                    [inputPath: inputPath, outputFilename: filename_ix]
+                  }
+                  def transposedOutputs = ["inputPath", "outputFilename"].collectEntries{ key -> 
+                    [key, outputPerFile.collect{dic -> dic[key]}]
+                  }
+                  return [[key: plainName_] + transposedOutputs]
+                } else {
+                  def value_ = java.nio.file.Paths.get(filename)
+                  def inputPath = value instanceof File ? value.toPath() : value
+                  return [[inputPath: [inputPath], outputFilename: [filename]]]
+                }
+              }
+          
+          def inputPaths = processedState.collectMany{it.inputPath}
+          def outputFilenames = processedState.collectMany{it.outputFilename}
+          
+
+          [id_, inputPaths, outputFilenames]
+        }
+        | publishFilesProc
+    emit: input_ch
+  }
+  return publishFilesSimpleWf
+}
+
+
+
+
 // helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishStates.nf'
 def collectFiles(obj) {
   if (obj instanceof java.io.File || obj instanceof Path)  {
@@ -1723,8 +1877,6 @@ def publishStates(Map args) {
 
           // the input files and the target output filenames
           def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose()
-          def inputFiles_ = inputoutputFilenames_[0]
-          def outputFilenames_ = inputoutputFilenames_[1]
 
           def yamlFilename = yamlTemplate_
             .replaceAll('\\$id', id_)
@@ -1737,7 +1889,7 @@ def publishStates(Map args) {
           // convert state to yaml blob
           def yamlBlob_ = toRelativeTaggedYamlBlob([id: id_] + state_, java.nio.file.Paths.get(yamlFilename))
 
-          [id_, yamlBlob_, yamlFilename, inputFiles_, outputFilenames_]
+          [id_, yamlBlob_, yamlFilename]
         }
         | publishStatesProc
     emit: input_ch
@@ -1749,33 +1901,17 @@ process publishStatesProc {
   publishDir path: "${getPublishDir()}/", mode: "copy"
   tag "$id"
   input:
-    tuple val(id), val(yamlBlob), val(yamlFile), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles)
+    tuple val(id), val(yamlBlob), val(yamlFile)
   output:
-    tuple val(id), path{[yamlFile] + outputFiles}
+    tuple val(id), path{[yamlFile]}
   script:
-  def copyCommands = [
-    inputFiles instanceof List ? inputFiles : [inputFiles],
-    outputFiles instanceof List ? outputFiles : [outputFiles]
-  ]
-    .transpose()
-    .collectMany{infile, outfile ->
-      if (infile.toString() != outfile.toString()) {
-        [
-          "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"",
-          "cp -r '${infile.toString()}' '${outfile.toString()}'"
-        ]
-      } else {
-        // no need to copy if infile is the same as outfile
-        []
-      }
-    }
   """
-mkdir -p "\$(dirname '${yamlFile}')"
+  mkdir -p "\$(dirname '${yamlFile}')"
-echo "Storing state as yaml"
+  echo "Storing state as yaml"
-echo '${yamlBlob}' > '${yamlFile}'
+  cat > '${yamlFile}' << HERE
-echo "Copying output files to destination folder"
+${yamlBlob}
-${copyCommands.join("\n  ")}
+HERE
-"""
+  """
 }
 
 
@@ -1806,13 +1942,10 @@ def publishStatesByConfig(Map args) {
             .replaceAll('\\$\\{key\\}', key_)
           def yamlDir = java.nio.file.Paths.get(yamlFilename).getParent()
 
-          // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where
+          // the processed state is a list of [key, value] tuples, where
           //   - key is a String
           //   - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path)
-          //   - inputPath is a List[Path]
-          //   - outputFilename is a List[String]
           //   - (key, value) are the tuples that will be saved to the state.yaml file
-          //   - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml)
           def processedState =
             config.allArguments
               .findAll { it.direction == "output" }
@@ -1829,7 +1962,7 @@ def publishStatesByConfig(Map args) {
                 // in the state as-is, but is not something that needs 
                 // to be copied from the source path to the dest path
                 if (par.type != "file") {
-                  return [[key: plainName_, value: value, inputPath: [], outputFilename: []]]
+                  return [[key: plainName_, value: value]]
                 }
                 // if the orig state does not contain this filename,
                 // it's an optional argument for which the user specified
@@ -1860,13 +1993,9 @@ def publishStatesByConfig(Map args) {
                     if (yamlDir != null) {
                       value_ = yamlDir.relativize(value_)
                     }
-                    def inputPath = val instanceof File ? val.toPath() : val
+                    return value_
-                    [value: value_, inputPath: inputPath, outputFilename: filename_ix]
                   }
-                  def transposedOutputs = ["value", "inputPath", "outputFilename"].collectEntries{ key -> 
+                  return [["key": plainName_, "value": outputPerFile]]
-                    [key, outputPerFile.collect{dic -> dic[key]}]
-                  }
-                  return [[key: plainName_] + transposedOutputs]
                 } else {
                   def value_ = java.nio.file.Paths.get(filename)
                   // if id contains a slash
@@ -1874,18 +2003,17 @@ def publishStatesByConfig(Map args) {
                     value_ = yamlDir.relativize(value_)
                   }
                   def inputPath = value instanceof File ? value.toPath() : value
-                  return [[key: plainName_, value: value_, inputPath: [inputPath], outputFilename: [filename]]]
+                  return [["key": plainName_, value: value_]]
                 }
               }
+              
           
           def updatedState_ = processedState.collectEntries{[it.key, it.value]}
-          def inputPaths = processedState.collectMany{it.inputPath}
-          def outputFilenames = processedState.collectMany{it.outputFilename}
           
           // convert state to yaml blob
           def yamlBlob_ = toTaggedYamlBlob([id: id_] + updatedState_)
 
-          [id_, yamlBlob_, yamlFilename, inputPaths, outputFilenames]
+          [id_, yamlBlob_, yamlFilename]
         }
         | publishStatesProc
     emit: input_ch
@@ -2559,7 +2687,8 @@ def _debug(workflowArgs, debugKey) {
 def workflowFactory(Map args, Map defaultWfArgs, Map meta) {
   def workflowArgs = processWorkflowArgs(args, defaultWfArgs, meta)
   def key_ = workflowArgs["key"]
-  
+  def multipleArgs = meta.config.allArguments.findAll{ it.multiple }.collect{it.plainName}
+
   workflow workflowInstance {
     take: input_
 
@@ -2716,12 +2845,36 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) {
       }
 
     // TODO: move some of the _meta.join_id wrangling to the safeJoin() function.
-    def chInitialOutput = chArgsWithDefaults
+    def chInitialOutputMulti = chArgsWithDefaults
       | _debug(workflowArgs, "processed")
       // run workflow
       | innerWorkflowFactory(workflowArgs)
-      // check output tuple
+    def chInitialOutputList = chInitialOutputMulti instanceof List ? chInitialOutputMulti : [chInitialOutputMulti]
-      | map { id_, output_ ->
+    assert chInitialOutputList.size() > 0: "should have emitted at least one output channel"
+    // Add a channel ID to the events, which designates the channel the event was emitted from as a running number
+    // This number is used to sort the events later when the events are gathered from across the channels.
+    def chInitialOutputListWithIndexedEvents = chInitialOutputList.withIndex().collect{channel, channelIndex ->
+      def newChannel = channel
+        | map {tuple ->
+          assert tuple instanceof List : 
+          "Error in module '${key_}': element in output channel should be a tuple [id, data, ...otherargs...]\n" +
+          "  Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" +
+          "  Expected class: List. Found: tuple.getClass() is ${tuple.getClass()}"
+        
+          def newEvent = [channelIndex] + tuple
+          return newEvent
+        }
+      return newChannel
+    }
+    // Put the events into 1 channel, cover case where there is only one channel is emitted
+    def chInitialOutput = chInitialOutputList.size() > 1 ? \
+      chInitialOutputListWithIndexedEvents[0].mix(*chInitialOutputListWithIndexedEvents.tail()) : \
+      chInitialOutputListWithIndexedEvents[0]
+    def chInitialOutputProcessed = chInitialOutput
+      | map { tuple  ->
+        def channelId = tuple[0]
+        def id_ = tuple[1]
+        def output_ = tuple[2]
 
         // see if output map contains metadata
         def meta_ =
@@ -2734,19 +2887,94 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) {
         output_ = output_.findAll{k, v -> k != "_meta"}
 
         // check value types
-        output_ = _processOutputValues(output_, meta.config, id_, key_)
+        output_ = _checkValidOutputArgument(output_, meta.config, id_, key_)
 
-        // simplify output if need be
+        [join_id, channelId, id_, output_]
-        if (workflowArgs.auto.simplifyOutput && output_.size() == 1) {
-          output_ = output_.values()[0]
-        }
-
-        [join_id, id_, output_]
       }
       // | view{"chInitialOutput: ${it.take(3)}"}
 
+    // join the output [prev_id, channel_id, new_id, output] with the previous state [prev_id, state, ...]
+    def chPublishWithPreviousState = safeJoin(chInitialOutputProcessed, chRunFiltered, key_)
+      // input tuple format: [join_id, channel_id, id, output, prev_state, ...]
+      // output tuple format: [join_id, channel_id, id, new_state, ...]
+      | map{ tup ->
+        def new_state = workflowArgs.toState(tup.drop(2).take(3))
+        tup.take(3) + [new_state] + tup.drop(5)
+      }
+    if (workflowArgs.auto.publish == "state") {
+      def chPublishFiles = chPublishWithPreviousState
+        // input tuple format: [join_id, channel_id, id, new_state, ...]
+        // output tuple format: [join_id, channel_id, id, new_state]
+        | map{ tup ->
+          tup.take(4)
+        }
+
+      safeJoin(chPublishFiles, chArgsWithDefaults, key_)
+        // input tuple format: [join_id, channel_id, id, new_state, orig_state, ...]
+        // output tuple format: [id, new_state, orig_state]
+        | map { tup ->
+          tup.drop(2).take(3)
+        }
+        | publishFilesByConfig(key: key_, config: meta.config)
+    }
+    // Join the state from the events that were emitted from different channels
+    def chJoined = chInitialOutputProcessed
+      | map {tuple ->
+        def join_id = tuple[0]
+        def channel_id = tuple[1]
+        def id = tuple[2]
+        def other = tuple.drop(3)
+        // Below, groupTuple is used to join the events. To make sure resuming a workflow
+        // keeps working, the output state must be deterministic. This means the state needs to be
+        // sorted with groupTuple's has a 'sort' argument. This argument can be set to 'hash',
+        // but hashing the state when it is large can be problematic in terms of performance.
+        // Therefore, a custom comparator function is provided. We add the channel ID to the 
+        // states so that we can use the channel ID to sort the items. 
+        def stateWithChannelID = [[channel_id] * other.size(), other].transpose()
+        // A comparator that is provided to groupTuple's 'sort' argument is applied
+        // to all elements of the event tuple (that is not the 'id'). The comparator
+        // closure that is used below expects the input to be List. So the join_id and
+        // channel_id must also be wrapped in a list. 
+        [[join_id], [channel_id], id] + stateWithChannelID
+      }
+      | groupTuple(by: 2, sort: {a, b -> a[0] <=> b[0]}, size: chInitialOutputList.size(), remainder: true)
+      | map {join_ids, _, id, statesWithChannelID ->
+        // Remove the channel IDs from the states
+        def states = statesWithChannelID.collect{it[1]}
+        def newJoinId = join_ids.flatten().unique{a, b -> a <=> b}
+        assert newJoinId.size() == 1: "Multiple events were emitted for '$id'."
+        def newJoinIdUnique = newJoinId[0]
+        
+        // Merge the states from the different channels
+        def newState = states.inject([:]){ old_state, state_to_add ->
+          return old_state + state_to_add.collectEntries{k, v -> 
+            if (!multipleArgs.contains(k)) {
+              // if the key is not a multiple argument, we expect only one value
+              if (old_state.containsKey(k)) {
+                assert old_state[k] == v : "ID $id: multiple entries for argument $k were emitted."
+              }
+              [k, v]
+            } else {
+              // if the key is a multiple argument, append the different values into one list
+              def prevValue = old_state.getOrDefault(k, [])
+              def prevValueAsList = prevValue instanceof List ? prevValue : [prevValue]
+              [k, prevValueAsList + v]
+            }
+          }
+        }
+
+        _checkAllRequiredOuputsPresent(newState, meta.config, id, key_)
+
+        // simplify output if need be
+        if (workflowArgs.auto.simplifyOutput && newState.size() == 1) {
+          newState = newState.values()[0]
+        }
+
+        return [newJoinIdUnique, id, newState]
+      }
+    
     // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...]
-    def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_)
+    def chNewState = safeJoin(chJoined, chRunFiltered, key_)
       // input tuple format: [join_id, id, output, prev_state, ...]
       // output tuple format: [join_id, id, new_state, ...]
       | map{ tup ->
@@ -2755,23 +2983,21 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) {
       }
 
     if (workflowArgs.auto.publish == "state") {
-      def chPublish = chNewState
+      def chPublishStates = chNewState
         // input tuple format: [join_id, id, new_state, ...]
         // output tuple format: [join_id, id, new_state]
         | map{ tup ->
           tup.take(3)
         }
 
-      safeJoin(chPublish, chArgsWithDefaults, key_)
+      safeJoin(chPublishStates, chArgsWithDefaults, key_)
         // input tuple format: [join_id, id, new_state, orig_state, ...]
         // output tuple format: [id, new_state, orig_state]
         | map { tup ->
           tup.drop(1).take(3)
-      }
+        }
         | publishStatesByConfig(key: key_, config: meta.config)
     }
-
-    // remove join_id and meta
     chReturn = chNewState
       | map { tup ->
         // input tuple format: [join_id, id, new_state, ...]
@@ -2806,7 +3032,7 @@ meta = [
   "config": processConfig(readJsonBlob('''{
   "name" : "combine_samples",
   "namespace" : "dataflow",
-  "version" : "v0.3.7",
+  "version" : "v0.3.9",
   "argument_groups" : [
     {
       "name" : "Input arguments",
@@ -2903,6 +3129,10 @@ meta = [
   ],
   "description" : "Combine fastq files from across samples into one event with a list of fastq files per orientation.",
   "status" : "enabled",
+  "scope" : {
+    "image" : "public",
+    "target" : "public"
+  },
   "requirements" : {
     "commands" : [
       "ps"
@@ -2999,14 +3229,14 @@ meta = [
     "runner" : "nextflow",
     "engine" : "native|native",
     "output" : "target/nextflow/dataflow/combine_samples",
-    "viash_version" : "0.9.0",
+    "viash_version" : "0.9.4",
-    "git_commit" : "e177226c52ca1dd5a5cf9bb0bfc108812e2aa332",
+    "git_commit" : "820318c378d8119e2aa98768282e43c2aa017ba7",
     "git_remote" : "https://github.com/viash-hub/demultiplex",
-    "git_tag" : "v0.3.5-6-ge177226"
+    "git_tag" : "v0.3.8-5-g820318c"
   },
   "package_config" : {
     "name" : "demultiplex",
-    "version" : "v0.3.7",
+    "version" : "v0.3.9",
     "description" : "Demultiplexing pipeline\n",
     "info" : {
       "test_resources" : [
@@ -3016,14 +3246,14 @@ meta = [
         }
       ]
     },
-    "viash_version" : "0.9.0",
+    "viash_version" : "0.9.4",
     "source" : "src",
     "target" : "target",
     "config_mods" : [
-      ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n",
+      ".requirements.commands += ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n",
       ".engines += { type: \\"native\\" }",
       ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'",
-      ".engines[.type == 'docker'].target_tag := 'v0.3.7'"
+      ".engines[.type == 'docker'].target_tag := 'v0.3.9'"
     ],
     "keywords" : [
       "bioinformatics",

target/nextflow/dataflow/combine_samples/nextflow.config

@@ -2,7 +2,7 @@ manifest {
   name = 'dataflow/combine_samples'
   mainScript = 'main.nf'
   nextflowVersion = '!>=20.12.1-edge'
-  version = 'v0.3.7'
+  version = 'v0.3.9'
   description = 'Combine fastq files from across samples into one event with a list of fastq files per orientation.'
 }
 

target/nextflow/dataflow/combine_samples/nextflow_schema.json

@@ -67,10 +67,10 @@
                 "output_forward": {
                 "type":
                 "string",
-                "description": "Type: List of `file`, required, default: `$id.$key.output_forward_*.output_forward_*`, multiple_sep: `\";\"`. ",
+                "description": "Type: List of `file`, required, default: `$id.$key.output_forward_*`, multiple_sep: `\";\"`. ",
-                "help_text": "Type: List of `file`, required, default: `$id.$key.output_forward_*.output_forward_*`, multiple_sep: `\";\"`. "
+                "help_text": "Type: List of `file`, required, default: `$id.$key.output_forward_*`, multiple_sep: `\";\"`. "
             ,
-                "default":"$id.$key.output_forward_*.output_forward_*"
+                "default":"$id.$key.output_forward_*"
             }
     
 
@@ -78,10 +78,10 @@
                 "output_reverse": {
                 "type":
                 "string",
-                "description": "Type: List of `file`, default: `$id.$key.output_reverse_*.output_reverse_*`, multiple_sep: `\";\"`. ",
+                "description": "Type: List of `file`, default: `$id.$key.output_reverse_*`, multiple_sep: `\";\"`. ",
-                "help_text": "Type: List of `file`, default: `$id.$key.output_reverse_*.output_reverse_*`, multiple_sep: `\";\"`. "
+                "help_text": "Type: List of `file`, default: `$id.$key.output_reverse_*`, multiple_sep: `\";\"`. "
             ,
-                "default":"$id.$key.output_reverse_*.output_reverse_*"
+                "default":"$id.$key.output_reverse_*"
             }
     
 
@@ -89,10 +89,10 @@
                 "output_falco": {
                 "type":
                 "string",
-                "description": "Type: List of `file`, required, default: `$id.$key.output_falco_*.output_falco_*`, multiple_sep: `\";\"`. ",
+                "description": "Type: List of `file`, required, default: `$id.$key.output_falco_*`, multiple_sep: `\";\"`. ",
-                "help_text": "Type: List of `file`, required, default: `$id.$key.output_falco_*.output_falco_*`, multiple_sep: `\";\"`. "
+                "help_text": "Type: List of `file`, required, default: `$id.$key.output_falco_*`, multiple_sep: `\";\"`. "
             ,
-                "default":"$id.$key.output_falco_*.output_falco_*"
+                "default":"$id.$key.output_falco_*"
             }
     
 

target/nextflow/dataflow/gather_fastqs_and_validate/.config.vsh.yaml

@@ -1,6 +1,6 @@
 name: "gather_fastqs_and_validate"
 namespace: "dataflow"
-version: "v0.3.7"
+version: "v0.3.9"
 argument_groups:
 - name: "Input arguments"
   arguments:
@@ -56,6 +56,9 @@ description: "From a directory containing fastq files, gather the files per samp
   \ \nand validate according to the contents of the sample sheet.\n"
 info: null
 status: "enabled"
+scope:
+  image: "public"
+  target: "public"
 requirements:
   commands:
   - "ps"
@@ -137,29 +140,29 @@ build_info:
   engine: "native|native"
   output: "target/nextflow/dataflow/gather_fastqs_and_validate"
   executable: "target/nextflow/dataflow/gather_fastqs_and_validate/main.nf"
-  viash_version: "0.9.0"
+  viash_version: "0.9.4"
-  git_commit: "e177226c52ca1dd5a5cf9bb0bfc108812e2aa332"
+  git_commit: "820318c378d8119e2aa98768282e43c2aa017ba7"
   git_remote: "https://github.com/viash-hub/demultiplex"
-  git_tag: "v0.3.5-6-ge177226"
+  git_tag: "v0.3.8-5-g820318c"
 package_config:
   name: "demultiplex"
-  version: "v0.3.7"
+  version: "v0.3.9"
   description: "Demultiplexing pipeline\n"
   info:
     test_resources:
     - path: "gs://viash-hub-test-data/demultiplex/v2/"
       dest: "testData"
-  viash_version: "0.9.0"
+  viash_version: "0.9.4"
   source: "src"
   target: "target"
   config_mods:
-  - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\
+  - ".requirements.commands += ['ps']\n.runners[.type == 'nextflow'].directives.tag\
     \ := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n\
     .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\
     )'\n"
   - ".engines += { type: \"native\" }"
   - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
-  - ".engines[.type == 'docker'].target_tag := 'v0.3.7'"
+  - ".engines[.type == 'docker'].target_tag := 'v0.3.9'"
   keywords:
   - "bioinformatics"
   - "sequence"

target/nextflow/dataflow/gather_fastqs_and_validate/main.nf

@@ -1,6 +1,6 @@
-// gather_fastqs_and_validate v0.3.7
+// gather_fastqs_and_validate v0.3.9
 // 
-// This wrapper script is auto-generated by viash 0.9.0 and is thus a derivative
+// This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative
 // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data
 // Intuitive.
 // 
@@ -82,64 +82,56 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi
       foundClass = "List[${e.foundClass}]"
     }
   } else if (par.type == "string") {
-    // cast to string if need be
+    // cast to string if need be. only cast if the value is a GString
     if (value instanceof GString) {
-      value = value.toString()
+      value = value as String
     }
     expectedClass = value instanceof String ? null : "String"
   } else if (par.type == "integer") {
     // cast to integer if need be
-    if (value instanceof String) {
+    if (value !instanceof Integer) {
       try {
-        value = value.toInteger()
+        value = value as Integer
       } catch (NumberFormatException e) {
-        // do nothing
+        expectedClass = "Integer"
       }
     }
-    if (value instanceof java.math.BigInteger) {
-      value = value.intValue()
-    }
-    expectedClass = value instanceof Integer ? null : "Integer"
   } else if (par.type == "long") {
     // cast to long if need be
-    if (value instanceof String) {
+    if (value !instanceof Long) {
       try {
-        value = value.toLong()
+        value = value as Long
       } catch (NumberFormatException e) {
-        // do nothing
+        expectedClass = "Long"
       }
     }
-    if (value instanceof Integer) {
-      value = value.toLong()
-    }
-    expectedClass = value instanceof Long ? null : "Long"
   } else if (par.type == "double") {
     // cast to double if need be
-    if (value instanceof String) {
+    if (value !instanceof Double) {
       try {
-        value = value.toDouble()
+        value = value as Double
       } catch (NumberFormatException e) {
-        // do nothing
+        expectedClass = "Double"
       }
     }
-    if (value instanceof java.math.BigDecimal) {
+  } else if (par.type == "float") {
-      value = value.doubleValue()
+    // cast to float if need be
+    if (value !instanceof Float) {
+      try {
+        value = value as Float
+      } catch (NumberFormatException e) {
+        expectedClass = "Float"
+      }
     }
-    if (value instanceof Float) {
-      value = value.toDouble()
-    }
-    expectedClass = value instanceof Double ? null : "Double"
   } else if (par.type == "boolean" | par.type == "boolean_true" | par.type == "boolean_false") {
     // cast to boolean if need be
-    if (value instanceof String) {
+    if (value !instanceof Boolean) {
-      def valueLower = value.toLowerCase()
+      try {
-      if (valueLower == "true") {
+        value = value as Boolean
-        value = true
+      } catch (Exception e) {
-      } else if (valueLower == "false") {
+        expectedClass = "Boolean"
-        value = false
       }
     }
-    expectedClass = value instanceof Boolean ? null : "Boolean"
   } else if (par.type == "file" && (par.direction == "input" || stage == "output")) {
     // cast to path if need be
     if (value instanceof String) {
@@ -151,10 +143,13 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi
     expectedClass = value instanceof Path ? null : "Path"
   } else if (par.type == "file" && stage == "input" && par.direction == "output") {
     // cast to string if need be
-    if (value instanceof GString) {
+    if (value !instanceof String) {
-      value = value.toString()
+      try {
+        value = value as String
+      } catch (Exception e) {
+        expectedClass = "String"
+      }
     }
-    expectedClass = value instanceof String ? null : "String"
   } else {
     // didn't find a match for par.type
     expectedClass = par.type
@@ -173,7 +168,7 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi
 Map _processInputValues(Map inputs, Map config, String id, String key) {
   if (!workflow.stubRun) {
     config.allArguments.each { arg ->
-      if (arg.required) {
+      if (arg.required && arg.direction == "input") {
         assert inputs.containsKey(arg.plainName) && inputs.get(arg.plainName) != null : 
           "Error in module '${key}' id '${id}': required input argument '${arg.plainName}' is missing"
       }
@@ -192,15 +187,8 @@ Map _processInputValues(Map inputs, Map config, String id, String key) {
 }
 
 // helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_processOutputValues.nf'
-Map _processOutputValues(Map outputs, Map config, String id, String key) {
+Map _checkValidOutputArgument(Map outputs, Map config, String id, String key) {
   if (!workflow.stubRun) {
-    config.allArguments.each { arg ->
-      if (arg.direction == "output" && arg.required) {
-        assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : 
-          "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing"
-      }
-    }
-
     outputs = outputs.collectEntries { name, value ->
       def par = config.allArguments.find { it.plainName == name && it.direction == "output" }
       assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid output argument"
@@ -213,6 +201,16 @@ Map _processOutputValues(Map outputs, Map config, String id, String key) {
   return outputs
 }
 
+void _checkAllRequiredOuputsPresent(Map outputs, Map config, String id, String key) {
+  if (!workflow.stubRun) {
+    config.allArguments.each { arg ->
+      if (arg.direction == "output" && arg.required) {
+        assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : 
+          "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing"
+      }
+    }
+  }
+}
 // helper file: 'src/main/resources/io/viash/runners/nextflow/channel/IDChecker.nf'
 class IDChecker {
   final def items = [] as Set
@@ -1666,6 +1664,162 @@ def joinStates(Closure apply_) {
   }
   return joinStatesWf
 }
+// helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishFiles.nf'
+def publishFiles(Map args) {
+  def key_ = args.get("key")
+
+  assert key_ != null : "publishFiles: key must be specified"
+  
+  workflow publishFilesWf {
+    take: input_ch
+    main:
+      input_ch
+        | map { tup ->
+          def id_ = tup[0]
+          def state_ = tup[1]
+
+          // the input files and the target output filenames
+          def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose()
+          def inputFiles_ = inputoutputFilenames_[0]
+          def outputFilenames_ = inputoutputFilenames_[1]
+
+          [id_, inputFiles_, outputFilenames_]
+        }
+        | publishFilesProc
+    emit: input_ch
+  }
+  return publishFilesWf
+}
+
+process publishFilesProc {
+  // todo: check publishpath?
+  publishDir path: "${getPublishDir()}/", mode: "copy"
+  tag "$id"
+  input:
+    tuple val(id), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles)
+  output:
+    tuple val(id), path{outputFiles}
+  script:
+  def copyCommands = [
+    inputFiles instanceof List ? inputFiles : [inputFiles],
+    outputFiles instanceof List ? outputFiles : [outputFiles]
+  ]
+    .transpose()
+    .collectMany{infile, outfile ->
+      if (infile.toString() != outfile.toString()) {
+        [
+          "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"",
+          "cp -r '${infile.toString()}' '${outfile.toString()}'"
+        ]
+      } else {
+        // no need to copy if infile is the same as outfile
+        []
+      }
+    }
+  """
+  echo "Copying output files to destination folder"
+  ${copyCommands.join("\n  ")}
+  """
+}
+
+
+// this assumes that the state contains no other values other than those specified in the config
+def publishFilesByConfig(Map args) {
+  def config = args.get("config")
+  assert config != null : "publishFilesByConfig: config must be specified"
+
+  def key_ = args.get("key", config.name)
+  assert key_ != null : "publishFilesByConfig: key must be specified"
+  
+  workflow publishFilesSimpleWf {
+    take: input_ch
+    main:
+      input_ch
+        | map { tup ->
+          def id_ = tup[0]
+          def state_ = tup[1] // e.g. [output: new File("myoutput.h5ad"), k: 10]
+          def origState_ = tup[2] // e.g. [output: '$id.$key.foo.h5ad']
+
+
+          // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where
+          //   - key is a String
+          //   - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path)
+          //   - inputPath is a List[Path]
+          //   - outputFilename is a List[String]
+          //   - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml)
+          def processedState =
+            config.allArguments
+              .findAll { it.direction == "output" }
+              .collectMany { par ->
+                def plainName_ = par.plainName
+                // if the state does not contain the key, it's an
+                // optional argument for which the component did 
+                // not generate any output OR multiple channels were emitted
+                // and the output was just not added to using the channel
+                // that is now being parsed
+                if (!state_.containsKey(plainName_)) {
+                  return []
+                }
+                def value = state_[plainName_]
+                // if the parameter is not a file, it should be stored
+                // in the state as-is, but is not something that needs 
+                // to be copied from the source path to the dest path
+                if (par.type != "file") {
+                  return [[inputPath: [], outputFilename: []]]
+                }
+                // if the orig state does not contain this filename,
+                // it's an optional argument for which the user specified
+                // that it should not be returned as a state
+                if (!origState_.containsKey(plainName_)) {
+                  return []
+                }
+                def filenameTemplate = origState_[plainName_]
+                // if the pararameter is multiple: true, fetch the template
+                if (par.multiple && filenameTemplate instanceof List) {
+                  filenameTemplate = filenameTemplate[0]
+                }
+                // instantiate the template
+                def filename = filenameTemplate
+                  .replaceAll('\\$id', id_)
+                  .replaceAll('\\$\\{id\\}', id_)
+                  .replaceAll('\\$key', key_)
+                  .replaceAll('\\$\\{key\\}', key_)
+                if (par.multiple) {
+                  // if the parameter is multiple: true, the filename
+                  // should contain a wildcard '*' that is replaced with
+                  // the index of the file
+                  assert filename.contains("*") : "Module '${key_}' id '${id_}': Multiple output files specified, but no wildcard '*' in the filename: ${filename}"
+                  def outputPerFile = value.withIndex().collect{ val, ix ->
+                    def filename_ix = filename.replace("*", ix.toString())
+                    def inputPath = val instanceof File ? val.toPath() : val
+                    [inputPath: inputPath, outputFilename: filename_ix]
+                  }
+                  def transposedOutputs = ["inputPath", "outputFilename"].collectEntries{ key -> 
+                    [key, outputPerFile.collect{dic -> dic[key]}]
+                  }
+                  return [[key: plainName_] + transposedOutputs]
+                } else {
+                  def value_ = java.nio.file.Paths.get(filename)
+                  def inputPath = value instanceof File ? value.toPath() : value
+                  return [[inputPath: [inputPath], outputFilename: [filename]]]
+                }
+              }
+          
+          def inputPaths = processedState.collectMany{it.inputPath}
+          def outputFilenames = processedState.collectMany{it.outputFilename}
+          
+
+          [id_, inputPaths, outputFilenames]
+        }
+        | publishFilesProc
+    emit: input_ch
+  }
+  return publishFilesSimpleWf
+}
+
+
+
+
 // helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishStates.nf'
 def collectFiles(obj) {
   if (obj instanceof java.io.File || obj instanceof Path)  {
@@ -1723,8 +1877,6 @@ def publishStates(Map args) {
 
           // the input files and the target output filenames
           def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose()
-          def inputFiles_ = inputoutputFilenames_[0]
-          def outputFilenames_ = inputoutputFilenames_[1]
 
           def yamlFilename = yamlTemplate_
             .replaceAll('\\$id', id_)
@@ -1737,7 +1889,7 @@ def publishStates(Map args) {
           // convert state to yaml blob
           def yamlBlob_ = toRelativeTaggedYamlBlob([id: id_] + state_, java.nio.file.Paths.get(yamlFilename))
 
-          [id_, yamlBlob_, yamlFilename, inputFiles_, outputFilenames_]
+          [id_, yamlBlob_, yamlFilename]
         }
         | publishStatesProc
     emit: input_ch
@@ -1749,33 +1901,17 @@ process publishStatesProc {
   publishDir path: "${getPublishDir()}/", mode: "copy"
   tag "$id"
   input:
-    tuple val(id), val(yamlBlob), val(yamlFile), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles)
+    tuple val(id), val(yamlBlob), val(yamlFile)
   output:
-    tuple val(id), path{[yamlFile] + outputFiles}
+    tuple val(id), path{[yamlFile]}
   script:
-  def copyCommands = [
-    inputFiles instanceof List ? inputFiles : [inputFiles],
-    outputFiles instanceof List ? outputFiles : [outputFiles]
-  ]
-    .transpose()
-    .collectMany{infile, outfile ->
-      if (infile.toString() != outfile.toString()) {
-        [
-          "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"",
-          "cp -r '${infile.toString()}' '${outfile.toString()}'"
-        ]
-      } else {
-        // no need to copy if infile is the same as outfile
-        []
-      }
-    }
   """
-mkdir -p "\$(dirname '${yamlFile}')"
+  mkdir -p "\$(dirname '${yamlFile}')"
-echo "Storing state as yaml"
+  echo "Storing state as yaml"
-echo '${yamlBlob}' > '${yamlFile}'
+  cat > '${yamlFile}' << HERE
-echo "Copying output files to destination folder"
+${yamlBlob}
-${copyCommands.join("\n  ")}
+HERE
-"""
+  """
 }
 
 
@@ -1806,13 +1942,10 @@ def publishStatesByConfig(Map args) {
             .replaceAll('\\$\\{key\\}', key_)
           def yamlDir = java.nio.file.Paths.get(yamlFilename).getParent()
 
-          // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where
+          // the processed state is a list of [key, value] tuples, where
           //   - key is a String
           //   - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path)
-          //   - inputPath is a List[Path]
-          //   - outputFilename is a List[String]
           //   - (key, value) are the tuples that will be saved to the state.yaml file
-          //   - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml)
           def processedState =
             config.allArguments
               .findAll { it.direction == "output" }
@@ -1829,7 +1962,7 @@ def publishStatesByConfig(Map args) {
                 // in the state as-is, but is not something that needs 
                 // to be copied from the source path to the dest path
                 if (par.type != "file") {
-                  return [[key: plainName_, value: value, inputPath: [], outputFilename: []]]
+                  return [[key: plainName_, value: value]]
                 }
                 // if the orig state does not contain this filename,
                 // it's an optional argument for which the user specified
@@ -1860,13 +1993,9 @@ def publishStatesByConfig(Map args) {
                     if (yamlDir != null) {
                       value_ = yamlDir.relativize(value_)
                     }
-                    def inputPath = val instanceof File ? val.toPath() : val
+                    return value_
-                    [value: value_, inputPath: inputPath, outputFilename: filename_ix]
                   }
-                  def transposedOutputs = ["value", "inputPath", "outputFilename"].collectEntries{ key -> 
+                  return [["key": plainName_, "value": outputPerFile]]
-                    [key, outputPerFile.collect{dic -> dic[key]}]
-                  }
-                  return [[key: plainName_] + transposedOutputs]
                 } else {
                   def value_ = java.nio.file.Paths.get(filename)
                   // if id contains a slash
@@ -1874,18 +2003,17 @@ def publishStatesByConfig(Map args) {
                     value_ = yamlDir.relativize(value_)
                   }
                   def inputPath = value instanceof File ? value.toPath() : value
-                  return [[key: plainName_, value: value_, inputPath: [inputPath], outputFilename: [filename]]]
+                  return [["key": plainName_, value: value_]]
                 }
               }
+              
           
           def updatedState_ = processedState.collectEntries{[it.key, it.value]}
-          def inputPaths = processedState.collectMany{it.inputPath}
-          def outputFilenames = processedState.collectMany{it.outputFilename}
           
           // convert state to yaml blob
           def yamlBlob_ = toTaggedYamlBlob([id: id_] + updatedState_)
 
-          [id_, yamlBlob_, yamlFilename, inputPaths, outputFilenames]
+          [id_, yamlBlob_, yamlFilename]
         }
         | publishStatesProc
     emit: input_ch
@@ -2559,7 +2687,8 @@ def _debug(workflowArgs, debugKey) {
 def workflowFactory(Map args, Map defaultWfArgs, Map meta) {
   def workflowArgs = processWorkflowArgs(args, defaultWfArgs, meta)
   def key_ = workflowArgs["key"]
-  
+  def multipleArgs = meta.config.allArguments.findAll{ it.multiple }.collect{it.plainName}
+
   workflow workflowInstance {
     take: input_
 
@@ -2716,12 +2845,36 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) {
       }
 
     // TODO: move some of the _meta.join_id wrangling to the safeJoin() function.
-    def chInitialOutput = chArgsWithDefaults
+    def chInitialOutputMulti = chArgsWithDefaults
       | _debug(workflowArgs, "processed")
       // run workflow
       | innerWorkflowFactory(workflowArgs)
-      // check output tuple
+    def chInitialOutputList = chInitialOutputMulti instanceof List ? chInitialOutputMulti : [chInitialOutputMulti]
-      | map { id_, output_ ->
+    assert chInitialOutputList.size() > 0: "should have emitted at least one output channel"
+    // Add a channel ID to the events, which designates the channel the event was emitted from as a running number
+    // This number is used to sort the events later when the events are gathered from across the channels.
+    def chInitialOutputListWithIndexedEvents = chInitialOutputList.withIndex().collect{channel, channelIndex ->
+      def newChannel = channel
+        | map {tuple ->
+          assert tuple instanceof List : 
+          "Error in module '${key_}': element in output channel should be a tuple [id, data, ...otherargs...]\n" +
+          "  Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" +
+          "  Expected class: List. Found: tuple.getClass() is ${tuple.getClass()}"
+        
+          def newEvent = [channelIndex] + tuple
+          return newEvent
+        }
+      return newChannel
+    }
+    // Put the events into 1 channel, cover case where there is only one channel is emitted
+    def chInitialOutput = chInitialOutputList.size() > 1 ? \
+      chInitialOutputListWithIndexedEvents[0].mix(*chInitialOutputListWithIndexedEvents.tail()) : \
+      chInitialOutputListWithIndexedEvents[0]
+    def chInitialOutputProcessed = chInitialOutput
+      | map { tuple  ->
+        def channelId = tuple[0]
+        def id_ = tuple[1]
+        def output_ = tuple[2]
 
         // see if output map contains metadata
         def meta_ =
@@ -2734,19 +2887,94 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) {
         output_ = output_.findAll{k, v -> k != "_meta"}
 
         // check value types
-        output_ = _processOutputValues(output_, meta.config, id_, key_)
+        output_ = _checkValidOutputArgument(output_, meta.config, id_, key_)
 
-        // simplify output if need be
+        [join_id, channelId, id_, output_]
-        if (workflowArgs.auto.simplifyOutput && output_.size() == 1) {
-          output_ = output_.values()[0]
-        }
-
-        [join_id, id_, output_]
       }
       // | view{"chInitialOutput: ${it.take(3)}"}
 
+    // join the output [prev_id, channel_id, new_id, output] with the previous state [prev_id, state, ...]
+    def chPublishWithPreviousState = safeJoin(chInitialOutputProcessed, chRunFiltered, key_)
+      // input tuple format: [join_id, channel_id, id, output, prev_state, ...]
+      // output tuple format: [join_id, channel_id, id, new_state, ...]
+      | map{ tup ->
+        def new_state = workflowArgs.toState(tup.drop(2).take(3))
+        tup.take(3) + [new_state] + tup.drop(5)
+      }
+    if (workflowArgs.auto.publish == "state") {
+      def chPublishFiles = chPublishWithPreviousState
+        // input tuple format: [join_id, channel_id, id, new_state, ...]
+        // output tuple format: [join_id, channel_id, id, new_state]
+        | map{ tup ->
+          tup.take(4)
+        }
+
+      safeJoin(chPublishFiles, chArgsWithDefaults, key_)
+        // input tuple format: [join_id, channel_id, id, new_state, orig_state, ...]
+        // output tuple format: [id, new_state, orig_state]
+        | map { tup ->
+          tup.drop(2).take(3)
+        }
+        | publishFilesByConfig(key: key_, config: meta.config)
+    }
+    // Join the state from the events that were emitted from different channels
+    def chJoined = chInitialOutputProcessed
+      | map {tuple ->
+        def join_id = tuple[0]
+        def channel_id = tuple[1]
+        def id = tuple[2]
+        def other = tuple.drop(3)
+        // Below, groupTuple is used to join the events. To make sure resuming a workflow
+        // keeps working, the output state must be deterministic. This means the state needs to be
+        // sorted with groupTuple's has a 'sort' argument. This argument can be set to 'hash',
+        // but hashing the state when it is large can be problematic in terms of performance.
+        // Therefore, a custom comparator function is provided. We add the channel ID to the 
+        // states so that we can use the channel ID to sort the items. 
+        def stateWithChannelID = [[channel_id] * other.size(), other].transpose()
+        // A comparator that is provided to groupTuple's 'sort' argument is applied
+        // to all elements of the event tuple (that is not the 'id'). The comparator
+        // closure that is used below expects the input to be List. So the join_id and
+        // channel_id must also be wrapped in a list. 
+        [[join_id], [channel_id], id] + stateWithChannelID
+      }
+      | groupTuple(by: 2, sort: {a, b -> a[0] <=> b[0]}, size: chInitialOutputList.size(), remainder: true)
+      | map {join_ids, _, id, statesWithChannelID ->
+        // Remove the channel IDs from the states
+        def states = statesWithChannelID.collect{it[1]}
+        def newJoinId = join_ids.flatten().unique{a, b -> a <=> b}
+        assert newJoinId.size() == 1: "Multiple events were emitted for '$id'."
+        def newJoinIdUnique = newJoinId[0]
+        
+        // Merge the states from the different channels
+        def newState = states.inject([:]){ old_state, state_to_add ->
+          return old_state + state_to_add.collectEntries{k, v -> 
+            if (!multipleArgs.contains(k)) {
+              // if the key is not a multiple argument, we expect only one value
+              if (old_state.containsKey(k)) {
+                assert old_state[k] == v : "ID $id: multiple entries for argument $k were emitted."
+              }
+              [k, v]
+            } else {
+              // if the key is a multiple argument, append the different values into one list
+              def prevValue = old_state.getOrDefault(k, [])
+              def prevValueAsList = prevValue instanceof List ? prevValue : [prevValue]
+              [k, prevValueAsList + v]
+            }
+          }
+        }
+
+        _checkAllRequiredOuputsPresent(newState, meta.config, id, key_)
+
+        // simplify output if need be
+        if (workflowArgs.auto.simplifyOutput && newState.size() == 1) {
+          newState = newState.values()[0]
+        }
+
+        return [newJoinIdUnique, id, newState]
+      }
+    
     // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...]
-    def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_)
+    def chNewState = safeJoin(chJoined, chRunFiltered, key_)
       // input tuple format: [join_id, id, output, prev_state, ...]
       // output tuple format: [join_id, id, new_state, ...]
       | map{ tup ->
@@ -2755,23 +2983,21 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) {
       }
 
     if (workflowArgs.auto.publish == "state") {
-      def chPublish = chNewState
+      def chPublishStates = chNewState
         // input tuple format: [join_id, id, new_state, ...]
         // output tuple format: [join_id, id, new_state]
         | map{ tup ->
           tup.take(3)
         }
 
-      safeJoin(chPublish, chArgsWithDefaults, key_)
+      safeJoin(chPublishStates, chArgsWithDefaults, key_)
         // input tuple format: [join_id, id, new_state, orig_state, ...]
         // output tuple format: [id, new_state, orig_state]
         | map { tup ->
           tup.drop(1).take(3)
-      }
+        }
         | publishStatesByConfig(key: key_, config: meta.config)
     }
-
-    // remove join_id and meta
     chReturn = chNewState
       | map { tup ->
         // input tuple format: [join_id, id, new_state, ...]
@@ -2806,7 +3032,7 @@ meta = [
   "config": processConfig(readJsonBlob('''{
   "name" : "gather_fastqs_and_validate",
   "namespace" : "dataflow",
-  "version" : "v0.3.7",
+  "version" : "v0.3.9",
   "argument_groups" : [
     {
       "name" : "Input arguments",
@@ -2876,6 +3102,10 @@ meta = [
   ],
   "description" : "From a directory containing fastq files, gather the files per sample \nand validate according to the contents of the sample sheet.\n",
   "status" : "enabled",
+  "scope" : {
+    "image" : "public",
+    "target" : "public"
+  },
   "requirements" : {
     "commands" : [
       "ps"
@@ -2972,14 +3202,14 @@ meta = [
     "runner" : "nextflow",
     "engine" : "native|native",
     "output" : "target/nextflow/dataflow/gather_fastqs_and_validate",
-    "viash_version" : "0.9.0",
+    "viash_version" : "0.9.4",
-    "git_commit" : "e177226c52ca1dd5a5cf9bb0bfc108812e2aa332",
+    "git_commit" : "820318c378d8119e2aa98768282e43c2aa017ba7",
     "git_remote" : "https://github.com/viash-hub/demultiplex",
-    "git_tag" : "v0.3.5-6-ge177226"
+    "git_tag" : "v0.3.8-5-g820318c"
   },
   "package_config" : {
     "name" : "demultiplex",
-    "version" : "v0.3.7",
+    "version" : "v0.3.9",
     "description" : "Demultiplexing pipeline\n",
     "info" : {
       "test_resources" : [
@@ -2989,14 +3219,14 @@ meta = [
         }
       ]
     },
-    "viash_version" : "0.9.0",
+    "viash_version" : "0.9.4",
     "source" : "src",
     "target" : "target",
     "config_mods" : [
-      ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n",
+      ".requirements.commands += ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n",
       ".engines += { type: \\"native\\" }",
       ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'",
-      ".engines[.type == 'docker'].target_tag := 'v0.3.7'"
+      ".engines[.type == 'docker'].target_tag := 'v0.3.9'"
     ],
     "keywords" : [
       "bioinformatics",
@@ -3019,6 +3249,29 @@ meta = [
 
 // inner workflow
 // user-provided Nextflow code
+import java.util.zip.GZIPInputStream
+import java.nio.file.Files
+import java.io.BufferedInputStream
+
+def is_empty(file_to_check){
+  /* 
+  Checks if a file has content
+  */
+  if (file_to_check.size() == 0) {
+    return true
+  }
+  def input_stream = Files.newInputStream(file_to_check)
+  def gzInputStream
+  try {
+    gzInputStream = new GZIPInputStream(new BufferedInputStream(input_stream))
+  } catch (java.io.EOFException ex) {
+    // This is not a gzipfile...
+    return false
+  }
+  def read_one_byte = gzInputStream.read()
+  return read_one_byte == -1
+}
+
 workflow run_wf {
   take:
     input_ch
@@ -3099,6 +3352,9 @@ workflow run_wf {
             "Found forward: ${forward_fastq} and reverse: ${reverse_fastq}."
           println "Found ${forward_fastq.size()} forward and ${reverse_fastq.size()} reverse " +
             "fastq files for sample ${sample_id}"
+
+          assert forward_fastq.every{!is_empty(it)} && reverse_fastq.every{!is_empty(it)}:
+            "A fastq file for sample '${sample_id}' appears to be empty!"
           def fastqs_state = [
             "fastq_forward": forward_fastq,
             "fastq_reverse": reverse_fastq,

target/nextflow/dataflow/gather_fastqs_and_validate/nextflow.config

@@ -2,7 +2,7 @@ manifest {
   name = 'dataflow/gather_fastqs_and_validate'
   mainScript = 'main.nf'
   nextflowVersion = '!>=20.12.1-edge'
-  version = 'v0.3.7'
+  version = 'v0.3.9'
   description = 'From a directory containing fastq files, gather the files per sample \nand validate according to the contents of the sample sheet.\n'
 }
 

target/nextflow/dataflow/gather_fastqs_and_validate/nextflow_schema.json

@@ -47,10 +47,10 @@
                 "fastq_forward": {
                 "type":
                 "string",
-                "description": "Type: List of `file`, required, default: `$id.$key.fastq_forward_*.fastq_forward_*`, multiple_sep: `\";\"`. ",
+                "description": "Type: List of `file`, required, default: `$id.$key.fastq_forward_*`, multiple_sep: `\";\"`. ",
-                "help_text": "Type: List of `file`, required, default: `$id.$key.fastq_forward_*.fastq_forward_*`, multiple_sep: `\";\"`. "
+                "help_text": "Type: List of `file`, required, default: `$id.$key.fastq_forward_*`, multiple_sep: `\";\"`. "
             ,
-                "default":"$id.$key.fastq_forward_*.fastq_forward_*"
+                "default":"$id.$key.fastq_forward_*"
             }
     
 
@@ -58,10 +58,10 @@
                 "fastq_reverse": {
                 "type":
                 "string",
-                "description": "Type: List of `file`, default: `$id.$key.fastq_reverse_*.fastq_reverse_*`, multiple_sep: `\";\"`. ",
+                "description": "Type: List of `file`, default: `$id.$key.fastq_reverse_*`, multiple_sep: `\";\"`. ",
-                "help_text": "Type: List of `file`, default: `$id.$key.fastq_reverse_*.fastq_reverse_*`, multiple_sep: `\";\"`. "
+                "help_text": "Type: List of `file`, default: `$id.$key.fastq_reverse_*`, multiple_sep: `\";\"`. "
             ,
-                "default":"$id.$key.fastq_reverse_*.fastq_reverse_*"
+                "default":"$id.$key.fastq_reverse_*"
             }
     
 

target/nextflow/demultiplex/.config.vsh.yaml

@@ -1,5 +1,5 @@
 name: "demultiplex"
-version: "v0.3.7"
+version: "v0.3.9"
 argument_groups:
 - name: "Input arguments"
   arguments:
@@ -124,6 +124,9 @@ test_resources:
   entrypoint: "test_bases2fastq"
 info: null
 status: "enabled"
+scope:
+  image: "public"
+  target: "public"
 requirements:
   commands:
   - "ps"
@@ -243,10 +246,10 @@ build_info:
   engine: "native|native"
   output: "target/nextflow/demultiplex"
   executable: "target/nextflow/demultiplex/main.nf"
-  viash_version: "0.9.0"
+  viash_version: "0.9.4"
-  git_commit: "e177226c52ca1dd5a5cf9bb0bfc108812e2aa332"
+  git_commit: "820318c378d8119e2aa98768282e43c2aa017ba7"
   git_remote: "https://github.com/viash-hub/demultiplex"
-  git_tag: "v0.3.5-6-ge177226"
+  git_tag: "v0.3.8-5-g820318c"
   dependencies:
   - "target/nextflow/io/untar"
   - "target/nextflow/dataflow/gather_fastqs_and_validate"
@@ -258,23 +261,23 @@ build_info:
   - "target/dependencies/vsh/vsh/biobox/v0.3.0/nextflow/multiqc"
 package_config:
   name: "demultiplex"
-  version: "v0.3.7"
+  version: "v0.3.9"
   description: "Demultiplexing pipeline\n"
   info:
     test_resources:
     - path: "gs://viash-hub-test-data/demultiplex/v2/"
       dest: "testData"
-  viash_version: "0.9.0"
+  viash_version: "0.9.4"
   source: "src"
   target: "target"
   config_mods:
-  - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\
+  - ".requirements.commands += ['ps']\n.runners[.type == 'nextflow'].directives.tag\
     \ := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n\
     .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\
     )'\n"
   - ".engines += { type: \"native\" }"
   - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
-  - ".engines[.type == 'docker'].target_tag := 'v0.3.7'"
+  - ".engines[.type == 'docker'].target_tag := 'v0.3.9'"
   keywords:
   - "bioinformatics"
   - "sequence"

target/nextflow/demultiplex/main.nf

@@ -1,6 +1,6 @@
-// demultiplex v0.3.7
+// demultiplex v0.3.9
 // 
-// This wrapper script is auto-generated by viash 0.9.0 and is thus a derivative
+// This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative
 // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data
 // Intuitive.
 // 
@@ -82,64 +82,56 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi
       foundClass = "List[${e.foundClass}]"
     }
   } else if (par.type == "string") {
-    // cast to string if need be
+    // cast to string if need be. only cast if the value is a GString
     if (value instanceof GString) {
-      value = value.toString()
+      value = value as String
     }
     expectedClass = value instanceof String ? null : "String"
   } else if (par.type == "integer") {
     // cast to integer if need be
-    if (value instanceof String) {
+    if (value !instanceof Integer) {
       try {
-        value = value.toInteger()
+        value = value as Integer
       } catch (NumberFormatException e) {
-        // do nothing
+        expectedClass = "Integer"
       }
     }
-    if (value instanceof java.math.BigInteger) {
-      value = value.intValue()
-    }
-    expectedClass = value instanceof Integer ? null : "Integer"
   } else if (par.type == "long") {
     // cast to long if need be
-    if (value instanceof String) {
+    if (value !instanceof Long) {
       try {
-        value = value.toLong()
+        value = value as Long
       } catch (NumberFormatException e) {
-        // do nothing
+        expectedClass = "Long"
       }
     }
-    if (value instanceof Integer) {
-      value = value.toLong()
-    }
-    expectedClass = value instanceof Long ? null : "Long"
   } else if (par.type == "double") {
     // cast to double if need be
-    if (value instanceof String) {
+    if (value !instanceof Double) {
       try {
-        value = value.toDouble()
+        value = value as Double
       } catch (NumberFormatException e) {
-        // do nothing
+        expectedClass = "Double"
       }
     }
-    if (value instanceof java.math.BigDecimal) {
+  } else if (par.type == "float") {
-      value = value.doubleValue()
+    // cast to float if need be
+    if (value !instanceof Float) {
+      try {
+        value = value as Float
+      } catch (NumberFormatException e) {
+        expectedClass = "Float"
+      }
     }
-    if (value instanceof Float) {
-      value = value.toDouble()
-    }
-    expectedClass = value instanceof Double ? null : "Double"
   } else if (par.type == "boolean" | par.type == "boolean_true" | par.type == "boolean_false") {
     // cast to boolean if need be
-    if (value instanceof String) {
+    if (value !instanceof Boolean) {
-      def valueLower = value.toLowerCase()
+      try {
-      if (valueLower == "true") {
+        value = value as Boolean
-        value = true
+      } catch (Exception e) {
-      } else if (valueLower == "false") {
+        expectedClass = "Boolean"
-        value = false
       }
     }
-    expectedClass = value instanceof Boolean ? null : "Boolean"
   } else if (par.type == "file" && (par.direction == "input" || stage == "output")) {
     // cast to path if need be
     if (value instanceof String) {
@@ -151,10 +143,13 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi
     expectedClass = value instanceof Path ? null : "Path"
   } else if (par.type == "file" && stage == "input" && par.direction == "output") {
     // cast to string if need be
-    if (value instanceof GString) {
+    if (value !instanceof String) {
-      value = value.toString()
+      try {
+        value = value as String
+      } catch (Exception e) {
+        expectedClass = "String"
+      }
     }
-    expectedClass = value instanceof String ? null : "String"
   } else {
     // didn't find a match for par.type
     expectedClass = par.type
@@ -173,7 +168,7 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi
 Map _processInputValues(Map inputs, Map config, String id, String key) {
   if (!workflow.stubRun) {
     config.allArguments.each { arg ->
-      if (arg.required) {
+      if (arg.required && arg.direction == "input") {
         assert inputs.containsKey(arg.plainName) && inputs.get(arg.plainName) != null : 
           "Error in module '${key}' id '${id}': required input argument '${arg.plainName}' is missing"
       }
@@ -192,15 +187,8 @@ Map _processInputValues(Map inputs, Map config, String id, String key) {
 }
 
 // helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_processOutputValues.nf'
-Map _processOutputValues(Map outputs, Map config, String id, String key) {
+Map _checkValidOutputArgument(Map outputs, Map config, String id, String key) {
   if (!workflow.stubRun) {
-    config.allArguments.each { arg ->
-      if (arg.direction == "output" && arg.required) {
-        assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : 
-          "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing"
-      }
-    }
-
     outputs = outputs.collectEntries { name, value ->
       def par = config.allArguments.find { it.plainName == name && it.direction == "output" }
       assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid output argument"
@@ -213,6 +201,16 @@ Map _processOutputValues(Map outputs, Map config, String id, String key) {
   return outputs
 }
 
+void _checkAllRequiredOuputsPresent(Map outputs, Map config, String id, String key) {
+  if (!workflow.stubRun) {
+    config.allArguments.each { arg ->
+      if (arg.direction == "output" && arg.required) {
+        assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : 
+          "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing"
+      }
+    }
+  }
+}
 // helper file: 'src/main/resources/io/viash/runners/nextflow/channel/IDChecker.nf'
 class IDChecker {
   final def items = [] as Set
@@ -1666,6 +1664,162 @@ def joinStates(Closure apply_) {
   }
   return joinStatesWf
 }
+// helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishFiles.nf'
+def publishFiles(Map args) {
+  def key_ = args.get("key")
+
+  assert key_ != null : "publishFiles: key must be specified"
+  
+  workflow publishFilesWf {
+    take: input_ch
+    main:
+      input_ch
+        | map { tup ->
+          def id_ = tup[0]
+          def state_ = tup[1]
+
+          // the input files and the target output filenames
+          def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose()
+          def inputFiles_ = inputoutputFilenames_[0]
+          def outputFilenames_ = inputoutputFilenames_[1]
+
+          [id_, inputFiles_, outputFilenames_]
+        }
+        | publishFilesProc
+    emit: input_ch
+  }
+  return publishFilesWf
+}
+
+process publishFilesProc {
+  // todo: check publishpath?
+  publishDir path: "${getPublishDir()}/", mode: "copy"
+  tag "$id"
+  input:
+    tuple val(id), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles)
+  output:
+    tuple val(id), path{outputFiles}
+  script:
+  def copyCommands = [
+    inputFiles instanceof List ? inputFiles : [inputFiles],
+    outputFiles instanceof List ? outputFiles : [outputFiles]
+  ]
+    .transpose()
+    .collectMany{infile, outfile ->
+      if (infile.toString() != outfile.toString()) {
+        [
+          "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"",
+          "cp -r '${infile.toString()}' '${outfile.toString()}'"
+        ]
+      } else {
+        // no need to copy if infile is the same as outfile
+        []
+      }
+    }
+  """
+  echo "Copying output files to destination folder"
+  ${copyCommands.join("\n  ")}
+  """
+}
+
+
+// this assumes that the state contains no other values other than those specified in the config
+def publishFilesByConfig(Map args) {
+  def config = args.get("config")
+  assert config != null : "publishFilesByConfig: config must be specified"
+
+  def key_ = args.get("key", config.name)
+  assert key_ != null : "publishFilesByConfig: key must be specified"
+  
+  workflow publishFilesSimpleWf {
+    take: input_ch
+    main:
+      input_ch
+        | map { tup ->
+          def id_ = tup[0]
+          def state_ = tup[1] // e.g. [output: new File("myoutput.h5ad"), k: 10]
+          def origState_ = tup[2] // e.g. [output: '$id.$key.foo.h5ad']
+
+
+          // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where
+          //   - key is a String
+          //   - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path)
+          //   - inputPath is a List[Path]
+          //   - outputFilename is a List[String]
+          //   - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml)
+          def processedState =
+            config.allArguments
+              .findAll { it.direction == "output" }
+              .collectMany { par ->
+                def plainName_ = par.plainName
+                // if the state does not contain the key, it's an
+                // optional argument for which the component did 
+                // not generate any output OR multiple channels were emitted
+                // and the output was just not added to using the channel
+                // that is now being parsed
+                if (!state_.containsKey(plainName_)) {
+                  return []
+                }
+                def value = state_[plainName_]
+                // if the parameter is not a file, it should be stored
+                // in the state as-is, but is not something that needs 
+                // to be copied from the source path to the dest path
+                if (par.type != "file") {
+                  return [[inputPath: [], outputFilename: []]]
+                }
+                // if the orig state does not contain this filename,
+                // it's an optional argument for which the user specified
+                // that it should not be returned as a state
+                if (!origState_.containsKey(plainName_)) {
+                  return []
+                }
+                def filenameTemplate = origState_[plainName_]
+                // if the pararameter is multiple: true, fetch the template
+                if (par.multiple && filenameTemplate instanceof List) {
+                  filenameTemplate = filenameTemplate[0]
+                }
+                // instantiate the template
+                def filename = filenameTemplate
+                  .replaceAll('\\$id', id_)
+                  .replaceAll('\\$\\{id\\}', id_)
+                  .replaceAll('\\$key', key_)
+                  .replaceAll('\\$\\{key\\}', key_)
+                if (par.multiple) {
+                  // if the parameter is multiple: true, the filename
+                  // should contain a wildcard '*' that is replaced with
+                  // the index of the file
+                  assert filename.contains("*") : "Module '${key_}' id '${id_}': Multiple output files specified, but no wildcard '*' in the filename: ${filename}"
+                  def outputPerFile = value.withIndex().collect{ val, ix ->
+                    def filename_ix = filename.replace("*", ix.toString())
+                    def inputPath = val instanceof File ? val.toPath() : val
+                    [inputPath: inputPath, outputFilename: filename_ix]
+                  }
+                  def transposedOutputs = ["inputPath", "outputFilename"].collectEntries{ key -> 
+                    [key, outputPerFile.collect{dic -> dic[key]}]
+                  }
+                  return [[key: plainName_] + transposedOutputs]
+                } else {
+                  def value_ = java.nio.file.Paths.get(filename)
+                  def inputPath = value instanceof File ? value.toPath() : value
+                  return [[inputPath: [inputPath], outputFilename: [filename]]]
+                }
+              }
+          
+          def inputPaths = processedState.collectMany{it.inputPath}
+          def outputFilenames = processedState.collectMany{it.outputFilename}
+          
+
+          [id_, inputPaths, outputFilenames]
+        }
+        | publishFilesProc
+    emit: input_ch
+  }
+  return publishFilesSimpleWf
+}
+
+
+
+
 // helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishStates.nf'
 def collectFiles(obj) {
   if (obj instanceof java.io.File || obj instanceof Path)  {
@@ -1723,8 +1877,6 @@ def publishStates(Map args) {
 
           // the input files and the target output filenames
           def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose()
-          def inputFiles_ = inputoutputFilenames_[0]
-          def outputFilenames_ = inputoutputFilenames_[1]
 
           def yamlFilename = yamlTemplate_
             .replaceAll('\\$id', id_)
@@ -1737,7 +1889,7 @@ def publishStates(Map args) {
           // convert state to yaml blob
           def yamlBlob_ = toRelativeTaggedYamlBlob([id: id_] + state_, java.nio.file.Paths.get(yamlFilename))
 
-          [id_, yamlBlob_, yamlFilename, inputFiles_, outputFilenames_]
+          [id_, yamlBlob_, yamlFilename]
         }
         | publishStatesProc
     emit: input_ch
@@ -1749,33 +1901,17 @@ process publishStatesProc {
   publishDir path: "${getPublishDir()}/", mode: "copy"
   tag "$id"
   input:
-    tuple val(id), val(yamlBlob), val(yamlFile), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles)
+    tuple val(id), val(yamlBlob), val(yamlFile)
   output:
-    tuple val(id), path{[yamlFile] + outputFiles}
+    tuple val(id), path{[yamlFile]}
   script:
-  def copyCommands = [
-    inputFiles instanceof List ? inputFiles : [inputFiles],
-    outputFiles instanceof List ? outputFiles : [outputFiles]
-  ]
-    .transpose()
-    .collectMany{infile, outfile ->
-      if (infile.toString() != outfile.toString()) {
-        [
-          "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"",
-          "cp -r '${infile.toString()}' '${outfile.toString()}'"
-        ]
-      } else {
-        // no need to copy if infile is the same as outfile
-        []
-      }
-    }
   """
-mkdir -p "\$(dirname '${yamlFile}')"
+  mkdir -p "\$(dirname '${yamlFile}')"
-echo "Storing state as yaml"
+  echo "Storing state as yaml"
-echo '${yamlBlob}' > '${yamlFile}'
+  cat > '${yamlFile}' << HERE
-echo "Copying output files to destination folder"
+${yamlBlob}
-${copyCommands.join("\n  ")}
+HERE
-"""
+  """
 }
 
 
@@ -1806,13 +1942,10 @@ def publishStatesByConfig(Map args) {
             .replaceAll('\\$\\{key\\}', key_)
           def yamlDir = java.nio.file.Paths.get(yamlFilename).getParent()
 
-          // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where
+          // the processed state is a list of [key, value] tuples, where
           //   - key is a String
           //   - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path)
-          //   - inputPath is a List[Path]
-          //   - outputFilename is a List[String]
           //   - (key, value) are the tuples that will be saved to the state.yaml file
-          //   - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml)
           def processedState =
             config.allArguments
               .findAll { it.direction == "output" }
@@ -1829,7 +1962,7 @@ def publishStatesByConfig(Map args) {
                 // in the state as-is, but is not something that needs 
                 // to be copied from the source path to the dest path
                 if (par.type != "file") {
-                  return [[key: plainName_, value: value, inputPath: [], outputFilename: []]]
+                  return [[key: plainName_, value: value]]
                 }
                 // if the orig state does not contain this filename,
                 // it's an optional argument for which the user specified
@@ -1860,13 +1993,9 @@ def publishStatesByConfig(Map args) {
                     if (yamlDir != null) {
                       value_ = yamlDir.relativize(value_)
                     }
-                    def inputPath = val instanceof File ? val.toPath() : val
+                    return value_
-                    [value: value_, inputPath: inputPath, outputFilename: filename_ix]
                   }
-                  def transposedOutputs = ["value", "inputPath", "outputFilename"].collectEntries{ key -> 
+                  return [["key": plainName_, "value": outputPerFile]]
-                    [key, outputPerFile.collect{dic -> dic[key]}]
-                  }
-                  return [[key: plainName_] + transposedOutputs]
                 } else {
                   def value_ = java.nio.file.Paths.get(filename)
                   // if id contains a slash
@@ -1874,18 +2003,17 @@ def publishStatesByConfig(Map args) {
                     value_ = yamlDir.relativize(value_)
                   }
                   def inputPath = value instanceof File ? value.toPath() : value
-                  return [[key: plainName_, value: value_, inputPath: [inputPath], outputFilename: [filename]]]
+                  return [["key": plainName_, value: value_]]
                 }
               }
+              
           
           def updatedState_ = processedState.collectEntries{[it.key, it.value]}
-          def inputPaths = processedState.collectMany{it.inputPath}
-          def outputFilenames = processedState.collectMany{it.outputFilename}
           
           // convert state to yaml blob
           def yamlBlob_ = toTaggedYamlBlob([id: id_] + updatedState_)
 
-          [id_, yamlBlob_, yamlFilename, inputPaths, outputFilenames]
+          [id_, yamlBlob_, yamlFilename]
         }
         | publishStatesProc
     emit: input_ch
@@ -2559,7 +2687,8 @@ def _debug(workflowArgs, debugKey) {
 def workflowFactory(Map args, Map defaultWfArgs, Map meta) {
   def workflowArgs = processWorkflowArgs(args, defaultWfArgs, meta)
   def key_ = workflowArgs["key"]
-  
+  def multipleArgs = meta.config.allArguments.findAll{ it.multiple }.collect{it.plainName}
+
   workflow workflowInstance {
     take: input_
 
@@ -2716,12 +2845,36 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) {
       }
 
     // TODO: move some of the _meta.join_id wrangling to the safeJoin() function.
-    def chInitialOutput = chArgsWithDefaults
+    def chInitialOutputMulti = chArgsWithDefaults
       | _debug(workflowArgs, "processed")
       // run workflow
       | innerWorkflowFactory(workflowArgs)
-      // check output tuple
+    def chInitialOutputList = chInitialOutputMulti instanceof List ? chInitialOutputMulti : [chInitialOutputMulti]
-      | map { id_, output_ ->
+    assert chInitialOutputList.size() > 0: "should have emitted at least one output channel"
+    // Add a channel ID to the events, which designates the channel the event was emitted from as a running number
+    // This number is used to sort the events later when the events are gathered from across the channels.
+    def chInitialOutputListWithIndexedEvents = chInitialOutputList.withIndex().collect{channel, channelIndex ->
+      def newChannel = channel
+        | map {tuple ->
+          assert tuple instanceof List : 
+          "Error in module '${key_}': element in output channel should be a tuple [id, data, ...otherargs...]\n" +
+          "  Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" +
+          "  Expected class: List. Found: tuple.getClass() is ${tuple.getClass()}"
+        
+          def newEvent = [channelIndex] + tuple
+          return newEvent
+        }
+      return newChannel
+    }
+    // Put the events into 1 channel, cover case where there is only one channel is emitted
+    def chInitialOutput = chInitialOutputList.size() > 1 ? \
+      chInitialOutputListWithIndexedEvents[0].mix(*chInitialOutputListWithIndexedEvents.tail()) : \
+      chInitialOutputListWithIndexedEvents[0]
+    def chInitialOutputProcessed = chInitialOutput
+      | map { tuple  ->
+        def channelId = tuple[0]
+        def id_ = tuple[1]
+        def output_ = tuple[2]
 
         // see if output map contains metadata
         def meta_ =
@@ -2734,19 +2887,94 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) {
         output_ = output_.findAll{k, v -> k != "_meta"}
 
         // check value types
-        output_ = _processOutputValues(output_, meta.config, id_, key_)
+        output_ = _checkValidOutputArgument(output_, meta.config, id_, key_)
 
-        // simplify output if need be
+        [join_id, channelId, id_, output_]
-        if (workflowArgs.auto.simplifyOutput && output_.size() == 1) {
-          output_ = output_.values()[0]
-        }
-
-        [join_id, id_, output_]
       }
       // | view{"chInitialOutput: ${it.take(3)}"}
 
+    // join the output [prev_id, channel_id, new_id, output] with the previous state [prev_id, state, ...]
+    def chPublishWithPreviousState = safeJoin(chInitialOutputProcessed, chRunFiltered, key_)
+      // input tuple format: [join_id, channel_id, id, output, prev_state, ...]
+      // output tuple format: [join_id, channel_id, id, new_state, ...]
+      | map{ tup ->
+        def new_state = workflowArgs.toState(tup.drop(2).take(3))
+        tup.take(3) + [new_state] + tup.drop(5)
+      }
+    if (workflowArgs.auto.publish == "state") {
+      def chPublishFiles = chPublishWithPreviousState
+        // input tuple format: [join_id, channel_id, id, new_state, ...]
+        // output tuple format: [join_id, channel_id, id, new_state]
+        | map{ tup ->
+          tup.take(4)
+        }
+
+      safeJoin(chPublishFiles, chArgsWithDefaults, key_)
+        // input tuple format: [join_id, channel_id, id, new_state, orig_state, ...]
+        // output tuple format: [id, new_state, orig_state]
+        | map { tup ->
+          tup.drop(2).take(3)
+        }
+        | publishFilesByConfig(key: key_, config: meta.config)
+    }
+    // Join the state from the events that were emitted from different channels
+    def chJoined = chInitialOutputProcessed
+      | map {tuple ->
+        def join_id = tuple[0]
+        def channel_id = tuple[1]
+        def id = tuple[2]
+        def other = tuple.drop(3)
+        // Below, groupTuple is used to join the events. To make sure resuming a workflow
+        // keeps working, the output state must be deterministic. This means the state needs to be
+        // sorted with groupTuple's has a 'sort' argument. This argument can be set to 'hash',
+        // but hashing the state when it is large can be problematic in terms of performance.
+        // Therefore, a custom comparator function is provided. We add the channel ID to the 
+        // states so that we can use the channel ID to sort the items. 
+        def stateWithChannelID = [[channel_id] * other.size(), other].transpose()
+        // A comparator that is provided to groupTuple's 'sort' argument is applied
+        // to all elements of the event tuple (that is not the 'id'). The comparator
+        // closure that is used below expects the input to be List. So the join_id and
+        // channel_id must also be wrapped in a list. 
+        [[join_id], [channel_id], id] + stateWithChannelID
+      }
+      | groupTuple(by: 2, sort: {a, b -> a[0] <=> b[0]}, size: chInitialOutputList.size(), remainder: true)
+      | map {join_ids, _, id, statesWithChannelID ->
+        // Remove the channel IDs from the states
+        def states = statesWithChannelID.collect{it[1]}
+        def newJoinId = join_ids.flatten().unique{a, b -> a <=> b}
+        assert newJoinId.size() == 1: "Multiple events were emitted for '$id'."
+        def newJoinIdUnique = newJoinId[0]
+        
+        // Merge the states from the different channels
+        def newState = states.inject([:]){ old_state, state_to_add ->
+          return old_state + state_to_add.collectEntries{k, v -> 
+            if (!multipleArgs.contains(k)) {
+              // if the key is not a multiple argument, we expect only one value
+              if (old_state.containsKey(k)) {
+                assert old_state[k] == v : "ID $id: multiple entries for argument $k were emitted."
+              }
+              [k, v]
+            } else {
+              // if the key is a multiple argument, append the different values into one list
+              def prevValue = old_state.getOrDefault(k, [])
+              def prevValueAsList = prevValue instanceof List ? prevValue : [prevValue]
+              [k, prevValueAsList + v]
+            }
+          }
+        }
+
+        _checkAllRequiredOuputsPresent(newState, meta.config, id, key_)
+
+        // simplify output if need be
+        if (workflowArgs.auto.simplifyOutput && newState.size() == 1) {
+          newState = newState.values()[0]
+        }
+
+        return [newJoinIdUnique, id, newState]
+      }
+    
     // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...]
-    def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_)
+    def chNewState = safeJoin(chJoined, chRunFiltered, key_)
       // input tuple format: [join_id, id, output, prev_state, ...]
       // output tuple format: [join_id, id, new_state, ...]
       | map{ tup ->
@@ -2755,23 +2983,21 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) {
       }
 
     if (workflowArgs.auto.publish == "state") {
-      def chPublish = chNewState
+      def chPublishStates = chNewState
         // input tuple format: [join_id, id, new_state, ...]
         // output tuple format: [join_id, id, new_state]
         | map{ tup ->
           tup.take(3)
         }
 
-      safeJoin(chPublish, chArgsWithDefaults, key_)
+      safeJoin(chPublishStates, chArgsWithDefaults, key_)
         // input tuple format: [join_id, id, new_state, orig_state, ...]
         // output tuple format: [id, new_state, orig_state]
         | map { tup ->
           tup.drop(1).take(3)
-      }
+        }
         | publishStatesByConfig(key: key_, config: meta.config)
     }
-
-    // remove join_id and meta
     chReturn = chNewState
       | map { tup ->
         // input tuple format: [join_id, id, new_state, ...]
@@ -2805,7 +3031,7 @@ meta = [
   "resources_dir": moduleDir.toRealPath().normalize(),
   "config": processConfig(readJsonBlob('''{
   "name" : "demultiplex",
-  "version" : "v0.3.7",
+  "version" : "v0.3.9",
   "argument_groups" : [
     {
       "name" : "Input arguments",
@@ -2957,6 +3183,10 @@ meta = [
     }
   ],
   "status" : "enabled",
+  "scope" : {
+    "image" : "public",
+    "target" : "public"
+  },
   "requirements" : {
     "commands" : [
       "ps"
@@ -3119,14 +3349,14 @@ meta = [
     "runner" : "nextflow",
     "engine" : "native|native",
     "output" : "target/nextflow/demultiplex",
-    "viash_version" : "0.9.0",
+    "viash_version" : "0.9.4",
-    "git_commit" : "e177226c52ca1dd5a5cf9bb0bfc108812e2aa332",
+    "git_commit" : "820318c378d8119e2aa98768282e43c2aa017ba7",
     "git_remote" : "https://github.com/viash-hub/demultiplex",
-    "git_tag" : "v0.3.5-6-ge177226"
+    "git_tag" : "v0.3.8-5-g820318c"
   },
   "package_config" : {
     "name" : "demultiplex",
-    "version" : "v0.3.7",
+    "version" : "v0.3.9",
     "description" : "Demultiplexing pipeline\n",
     "info" : {
       "test_resources" : [
@@ -3136,14 +3366,14 @@ meta = [
         }
       ]
     },
-    "viash_version" : "0.9.0",
+    "viash_version" : "0.9.4",
     "source" : "src",
     "target" : "target",
     "config_mods" : [
-      ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n",
+      ".requirements.commands += ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n",
       ".engines += { type: \\"native\\" }",
       ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'",
-      ".engines[.type == 'docker'].target_tag := 'v0.3.7'"
+      ".engines[.type == 'docker'].target_tag := 'v0.3.9'"
     ],
     "keywords" : [
       "bioinformatics",

target/nextflow/demultiplex/nextflow.config

@@ -2,7 +2,7 @@ manifest {
   name = 'demultiplex'
   mainScript = 'main.nf'
   nextflowVersion = '!>=20.12.1-edge'
-  version = 'v0.3.7'
+  version = 'v0.3.9'
   description = 'Demultiplexing of raw sequencing data'
 }
 

target/nextflow/demultiplex/nextflow_schema.json

@@ -69,10 +69,10 @@
                 "output": {
                 "type":
                 "string",
-                "description": "Type: `file`, default: `$id.$key.output.output`. Directory to write fastq data to",
+                "description": "Type: `file`, default: `$id/fastq`. Directory to write fastq data to",
-                "help_text": "Type: `file`, default: `$id.$key.output.output`. Directory to write fastq data to"
+                "help_text": "Type: `file`, default: `$id/fastq`. Directory to write fastq data to"
             ,
-                "default":"$id.$key.output.output"
+                "default":"$id/fastq"
             }
     
 
@@ -80,10 +80,10 @@
                 "output_falco": {
                 "type":
                 "string",
-                "description": "Type: List of `file`, default: `$id.$key.output_falco_*.output_falco_*`, multiple_sep: `\";\"`. Directory to write falco output to",
+                "description": "Type: List of `file`, default: `$id/qc/fastqc`, multiple_sep: `\";\"`. Directory to write falco output to",
-                "help_text": "Type: List of `file`, default: `$id.$key.output_falco_*.output_falco_*`, multiple_sep: `\";\"`. Directory to write falco output to"
+                "help_text": "Type: List of `file`, default: `$id/qc/fastqc`, multiple_sep: `\";\"`. Directory to write falco output to"
             ,
-                "default":"$id.$key.output_falco_*.output_falco_*"
+                "default":"$id/qc/fastqc"
             }
     
 
@@ -91,10 +91,10 @@
                 "output_multiqc": {
                 "type":
                 "string",
-                "description": "Type: `file`, default: `$id.$key.output_multiqc.html`. Directory to write falco output to",
+                "description": "Type: `file`, default: `$id/qc/multiqc_report.html`. Directory to write falco output to",
-                "help_text": "Type: `file`, default: `$id.$key.output_multiqc.html`. Directory to write falco output to"
+                "help_text": "Type: `file`, default: `$id/qc/multiqc_report.html`. Directory to write falco output to"
             ,
-                "default":"$id.$key.output_multiqc.html"
+                "default":"$id/qc/multiqc_report.html"
             }
     
 
@@ -102,10 +102,10 @@
                 "output_run_information": {
                 "type":
                 "string",
-                "description": "Type: `file`, required, default: `$id.$key.output_run_information.csv`. ",
+                "description": "Type: `file`, required, default: `$id/run_information.csv`. ",
-                "help_text": "Type: `file`, required, default: `$id.$key.output_run_information.csv`. "
+                "help_text": "Type: `file`, required, default: `$id/run_information.csv`. "
             ,
-                "default":"$id.$key.output_run_information.csv"
+                "default":"$id/run_information.csv"
             }
     
 

target/nextflow/io/interop_summary_to_csv/.config.vsh.yaml

@@ -1,6 +1,6 @@
 name: "interop_summary_to_csv"
 namespace: "io"
-version: "v0.3.7"
+version: "v0.3.9"
 argument_groups:
 - name: "Input arguments"
   arguments:
@@ -43,8 +43,13 @@ resources:
   dest: "nextflow_labels.config"
 info: null
 status: "enabled"
+scope:
+  image: "public"
+  target: "public"
 requirements:
   commands:
+  - "summary"
+  - "index-summary"
   - "ps"
 license: "MIT"
 links:
@@ -121,7 +126,7 @@ engines:
   id: "docker"
   image: "debian:stable-slim"
   target_registry: "images.viash-hub.com"
-  target_tag: "v0.3.7"
+  target_tag: "v0.3.9"
   namespace_separator: "/"
   setup:
   - type: "apt"
@@ -145,29 +150,29 @@ build_info:
   engine: "docker|native"
   output: "target/nextflow/io/interop_summary_to_csv"
   executable: "target/nextflow/io/interop_summary_to_csv/main.nf"
-  viash_version: "0.9.0"
+  viash_version: "0.9.4"
-  git_commit: "e177226c52ca1dd5a5cf9bb0bfc108812e2aa332"
+  git_commit: "820318c378d8119e2aa98768282e43c2aa017ba7"
   git_remote: "https://github.com/viash-hub/demultiplex"
-  git_tag: "v0.3.5-6-ge177226"
+  git_tag: "v0.3.8-5-g820318c"
 package_config:
   name: "demultiplex"
-  version: "v0.3.7"
+  version: "v0.3.9"
   description: "Demultiplexing pipeline\n"
   info:
     test_resources:
     - path: "gs://viash-hub-test-data/demultiplex/v2/"
       dest: "testData"
-  viash_version: "0.9.0"
+  viash_version: "0.9.4"
   source: "src"
   target: "target"
   config_mods:
-  - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\
+  - ".requirements.commands += ['ps']\n.runners[.type == 'nextflow'].directives.tag\
     \ := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n\
     .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\
     )'\n"
   - ".engines += { type: \"native\" }"
   - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
-  - ".engines[.type == 'docker'].target_tag := 'v0.3.7'"
+  - ".engines[.type == 'docker'].target_tag := 'v0.3.9'"
   keywords:
   - "bioinformatics"
   - "sequence"

target/nextflow/io/interop_summary_to_csv/main.nf

@@ -1,6 +1,6 @@
-// interop_summary_to_csv v0.3.7
+// interop_summary_to_csv v0.3.9
 // 
-// This wrapper script is auto-generated by viash 0.9.0 and is thus a derivative
+// This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative
 // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data
 // Intuitive.
 // 
@@ -82,64 +82,56 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi
       foundClass = "List[${e.foundClass}]"
     }
   } else if (par.type == "string") {
-    // cast to string if need be
+    // cast to string if need be. only cast if the value is a GString
     if (value instanceof GString) {
-      value = value.toString()
+      value = value as String
     }
     expectedClass = value instanceof String ? null : "String"
   } else if (par.type == "integer") {
     // cast to integer if need be
-    if (value instanceof String) {
+    if (value !instanceof Integer) {
       try {
-        value = value.toInteger()
+        value = value as Integer
       } catch (NumberFormatException e) {
-        // do nothing
+        expectedClass = "Integer"
       }
     }
-    if (value instanceof java.math.BigInteger) {
-      value = value.intValue()
-    }
-    expectedClass = value instanceof Integer ? null : "Integer"
   } else if (par.type == "long") {
     // cast to long if need be
-    if (value instanceof String) {
+    if (value !instanceof Long) {
       try {
-        value = value.toLong()
+        value = value as Long
       } catch (NumberFormatException e) {
-        // do nothing
+        expectedClass = "Long"
       }
     }
-    if (value instanceof Integer) {
-      value = value.toLong()
-    }
-    expectedClass = value instanceof Long ? null : "Long"
   } else if (par.type == "double") {
     // cast to double if need be
-    if (value instanceof String) {
+    if (value !instanceof Double) {
       try {
-        value = value.toDouble()
+        value = value as Double
       } catch (NumberFormatException e) {
-        // do nothing
+        expectedClass = "Double"
       }
     }
-    if (value instanceof java.math.BigDecimal) {
+  } else if (par.type == "float") {
-      value = value.doubleValue()
+    // cast to float if need be
+    if (value !instanceof Float) {
+      try {
+        value = value as Float
+      } catch (NumberFormatException e) {
+        expectedClass = "Float"
+      }
     }
-    if (value instanceof Float) {
-      value = value.toDouble()
-    }
-    expectedClass = value instanceof Double ? null : "Double"
   } else if (par.type == "boolean" | par.type == "boolean_true" | par.type == "boolean_false") {
     // cast to boolean if need be
-    if (value instanceof String) {
+    if (value !instanceof Boolean) {
-      def valueLower = value.toLowerCase()
+      try {
-      if (valueLower == "true") {
+        value = value as Boolean
-        value = true
+      } catch (Exception e) {
-      } else if (valueLower == "false") {
+        expectedClass = "Boolean"
-        value = false
       }
     }
-    expectedClass = value instanceof Boolean ? null : "Boolean"
   } else if (par.type == "file" && (par.direction == "input" || stage == "output")) {
     // cast to path if need be
     if (value instanceof String) {
@@ -151,10 +143,13 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi
     expectedClass = value instanceof Path ? null : "Path"
   } else if (par.type == "file" && stage == "input" && par.direction == "output") {
     // cast to string if need be
-    if (value instanceof GString) {
+    if (value !instanceof String) {
-      value = value.toString()
+      try {
+        value = value as String
+      } catch (Exception e) {
+        expectedClass = "String"
+      }
     }
-    expectedClass = value instanceof String ? null : "String"
   } else {
     // didn't find a match for par.type
     expectedClass = par.type
@@ -173,7 +168,7 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi
 Map _processInputValues(Map inputs, Map config, String id, String key) {
   if (!workflow.stubRun) {
     config.allArguments.each { arg ->
-      if (arg.required) {
+      if (arg.required && arg.direction == "input") {
         assert inputs.containsKey(arg.plainName) && inputs.get(arg.plainName) != null : 
           "Error in module '${key}' id '${id}': required input argument '${arg.plainName}' is missing"
       }
@@ -192,15 +187,8 @@ Map _processInputValues(Map inputs, Map config, String id, String key) {
 }
 
 // helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_processOutputValues.nf'
-Map _processOutputValues(Map outputs, Map config, String id, String key) {
+Map _checkValidOutputArgument(Map outputs, Map config, String id, String key) {
   if (!workflow.stubRun) {
-    config.allArguments.each { arg ->
-      if (arg.direction == "output" && arg.required) {
-        assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : 
-          "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing"
-      }
-    }
-
     outputs = outputs.collectEntries { name, value ->
       def par = config.allArguments.find { it.plainName == name && it.direction == "output" }
       assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid output argument"
@@ -213,6 +201,16 @@ Map _processOutputValues(Map outputs, Map config, String id, String key) {
   return outputs
 }
 
+void _checkAllRequiredOuputsPresent(Map outputs, Map config, String id, String key) {
+  if (!workflow.stubRun) {
+    config.allArguments.each { arg ->
+      if (arg.direction == "output" && arg.required) {
+        assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : 
+          "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing"
+      }
+    }
+  }
+}
 // helper file: 'src/main/resources/io/viash/runners/nextflow/channel/IDChecker.nf'
 class IDChecker {
   final def items = [] as Set
@@ -1666,6 +1664,162 @@ def joinStates(Closure apply_) {
   }
   return joinStatesWf
 }
+// helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishFiles.nf'
+def publishFiles(Map args) {
+  def key_ = args.get("key")
+
+  assert key_ != null : "publishFiles: key must be specified"
+  
+  workflow publishFilesWf {
+    take: input_ch
+    main:
+      input_ch
+        | map { tup ->
+          def id_ = tup[0]
+          def state_ = tup[1]
+
+          // the input files and the target output filenames
+          def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose()
+          def inputFiles_ = inputoutputFilenames_[0]
+          def outputFilenames_ = inputoutputFilenames_[1]
+
+          [id_, inputFiles_, outputFilenames_]
+        }
+        | publishFilesProc
+    emit: input_ch
+  }
+  return publishFilesWf
+}
+
+process publishFilesProc {
+  // todo: check publishpath?
+  publishDir path: "${getPublishDir()}/", mode: "copy"
+  tag "$id"
+  input:
+    tuple val(id), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles)
+  output:
+    tuple val(id), path{outputFiles}
+  script:
+  def copyCommands = [
+    inputFiles instanceof List ? inputFiles : [inputFiles],
+    outputFiles instanceof List ? outputFiles : [outputFiles]
+  ]
+    .transpose()
+    .collectMany{infile, outfile ->
+      if (infile.toString() != outfile.toString()) {
+        [
+          "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"",
+          "cp -r '${infile.toString()}' '${outfile.toString()}'"
+        ]
+      } else {
+        // no need to copy if infile is the same as outfile
+        []
+      }
+    }
+  """
+  echo "Copying output files to destination folder"
+  ${copyCommands.join("\n  ")}
+  """
+}
+
+
+// this assumes that the state contains no other values other than those specified in the config
+def publishFilesByConfig(Map args) {
+  def config = args.get("config")
+  assert config != null : "publishFilesByConfig: config must be specified"
+
+  def key_ = args.get("key", config.name)
+  assert key_ != null : "publishFilesByConfig: key must be specified"
+  
+  workflow publishFilesSimpleWf {
+    take: input_ch
+    main:
+      input_ch
+        | map { tup ->
+          def id_ = tup[0]
+          def state_ = tup[1] // e.g. [output: new File("myoutput.h5ad"), k: 10]
+          def origState_ = tup[2] // e.g. [output: '$id.$key.foo.h5ad']
+
+
+          // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where
+          //   - key is a String
+          //   - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path)
+          //   - inputPath is a List[Path]
+          //   - outputFilename is a List[String]
+          //   - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml)
+          def processedState =
+            config.allArguments
+              .findAll { it.direction == "output" }
+              .collectMany { par ->
+                def plainName_ = par.plainName
+                // if the state does not contain the key, it's an
+                // optional argument for which the component did 
+                // not generate any output OR multiple channels were emitted
+                // and the output was just not added to using the channel
+                // that is now being parsed
+                if (!state_.containsKey(plainName_)) {
+                  return []
+                }
+                def value = state_[plainName_]
+                // if the parameter is not a file, it should be stored
+                // in the state as-is, but is not something that needs 
+                // to be copied from the source path to the dest path
+                if (par.type != "file") {
+                  return [[inputPath: [], outputFilename: []]]
+                }
+                // if the orig state does not contain this filename,
+                // it's an optional argument for which the user specified
+                // that it should not be returned as a state
+                if (!origState_.containsKey(plainName_)) {
+                  return []
+                }
+                def filenameTemplate = origState_[plainName_]
+                // if the pararameter is multiple: true, fetch the template
+                if (par.multiple && filenameTemplate instanceof List) {
+                  filenameTemplate = filenameTemplate[0]
+                }
+                // instantiate the template
+                def filename = filenameTemplate
+                  .replaceAll('\\$id', id_)
+                  .replaceAll('\\$\\{id\\}', id_)
+                  .replaceAll('\\$key', key_)
+                  .replaceAll('\\$\\{key\\}', key_)
+                if (par.multiple) {
+                  // if the parameter is multiple: true, the filename
+                  // should contain a wildcard '*' that is replaced with
+                  // the index of the file
+                  assert filename.contains("*") : "Module '${key_}' id '${id_}': Multiple output files specified, but no wildcard '*' in the filename: ${filename}"
+                  def outputPerFile = value.withIndex().collect{ val, ix ->
+                    def filename_ix = filename.replace("*", ix.toString())
+                    def inputPath = val instanceof File ? val.toPath() : val
+                    [inputPath: inputPath, outputFilename: filename_ix]
+                  }
+                  def transposedOutputs = ["inputPath", "outputFilename"].collectEntries{ key -> 
+                    [key, outputPerFile.collect{dic -> dic[key]}]
+                  }
+                  return [[key: plainName_] + transposedOutputs]
+                } else {
+                  def value_ = java.nio.file.Paths.get(filename)
+                  def inputPath = value instanceof File ? value.toPath() : value
+                  return [[inputPath: [inputPath], outputFilename: [filename]]]
+                }
+              }
+          
+          def inputPaths = processedState.collectMany{it.inputPath}
+          def outputFilenames = processedState.collectMany{it.outputFilename}
+          
+
+          [id_, inputPaths, outputFilenames]
+        }
+        | publishFilesProc
+    emit: input_ch
+  }
+  return publishFilesSimpleWf
+}
+
+
+
+
 // helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishStates.nf'
 def collectFiles(obj) {
   if (obj instanceof java.io.File || obj instanceof Path)  {
@@ -1723,8 +1877,6 @@ def publishStates(Map args) {
 
           // the input files and the target output filenames
           def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose()
-          def inputFiles_ = inputoutputFilenames_[0]
-          def outputFilenames_ = inputoutputFilenames_[1]
 
           def yamlFilename = yamlTemplate_
             .replaceAll('\\$id', id_)
@@ -1737,7 +1889,7 @@ def publishStates(Map args) {
           // convert state to yaml blob
           def yamlBlob_ = toRelativeTaggedYamlBlob([id: id_] + state_, java.nio.file.Paths.get(yamlFilename))
 
-          [id_, yamlBlob_, yamlFilename, inputFiles_, outputFilenames_]
+          [id_, yamlBlob_, yamlFilename]
         }
         | publishStatesProc
     emit: input_ch
@@ -1749,33 +1901,17 @@ process publishStatesProc {
   publishDir path: "${getPublishDir()}/", mode: "copy"
   tag "$id"
   input:
-    tuple val(id), val(yamlBlob), val(yamlFile), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles)
+    tuple val(id), val(yamlBlob), val(yamlFile)
   output:
-    tuple val(id), path{[yamlFile] + outputFiles}
+    tuple val(id), path{[yamlFile]}
   script:
-  def copyCommands = [
-    inputFiles instanceof List ? inputFiles : [inputFiles],
-    outputFiles instanceof List ? outputFiles : [outputFiles]
-  ]
-    .transpose()
-    .collectMany{infile, outfile ->
-      if (infile.toString() != outfile.toString()) {
-        [
-          "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"",
-          "cp -r '${infile.toString()}' '${outfile.toString()}'"
-        ]
-      } else {
-        // no need to copy if infile is the same as outfile
-        []
-      }
-    }
   """
-mkdir -p "\$(dirname '${yamlFile}')"
+  mkdir -p "\$(dirname '${yamlFile}')"
-echo "Storing state as yaml"
+  echo "Storing state as yaml"
-echo '${yamlBlob}' > '${yamlFile}'
+  cat > '${yamlFile}' << HERE
-echo "Copying output files to destination folder"
+${yamlBlob}
-${copyCommands.join("\n  ")}
+HERE
-"""
+  """
 }
 
 
@@ -1806,13 +1942,10 @@ def publishStatesByConfig(Map args) {
             .replaceAll('\\$\\{key\\}', key_)
           def yamlDir = java.nio.file.Paths.get(yamlFilename).getParent()
 
-          // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where
+          // the processed state is a list of [key, value] tuples, where
           //   - key is a String
           //   - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path)
-          //   - inputPath is a List[Path]
-          //   - outputFilename is a List[String]
           //   - (key, value) are the tuples that will be saved to the state.yaml file
-          //   - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml)
           def processedState =
             config.allArguments
               .findAll { it.direction == "output" }
@@ -1829,7 +1962,7 @@ def publishStatesByConfig(Map args) {
                 // in the state as-is, but is not something that needs 
                 // to be copied from the source path to the dest path
                 if (par.type != "file") {
-                  return [[key: plainName_, value: value, inputPath: [], outputFilename: []]]
+                  return [[key: plainName_, value: value]]
                 }
                 // if the orig state does not contain this filename,
                 // it's an optional argument for which the user specified
@@ -1860,13 +1993,9 @@ def publishStatesByConfig(Map args) {
                     if (yamlDir != null) {
                       value_ = yamlDir.relativize(value_)
                     }
-                    def inputPath = val instanceof File ? val.toPath() : val
+                    return value_
-                    [value: value_, inputPath: inputPath, outputFilename: filename_ix]
                   }
-                  def transposedOutputs = ["value", "inputPath", "outputFilename"].collectEntries{ key -> 
+                  return [["key": plainName_, "value": outputPerFile]]
-                    [key, outputPerFile.collect{dic -> dic[key]}]
-                  }
-                  return [[key: plainName_] + transposedOutputs]
                 } else {
                   def value_ = java.nio.file.Paths.get(filename)
                   // if id contains a slash
@@ -1874,18 +2003,17 @@ def publishStatesByConfig(Map args) {
                     value_ = yamlDir.relativize(value_)
                   }
                   def inputPath = value instanceof File ? value.toPath() : value
-                  return [[key: plainName_, value: value_, inputPath: [inputPath], outputFilename: [filename]]]
+                  return [["key": plainName_, value: value_]]
                 }
               }
+              
           
           def updatedState_ = processedState.collectEntries{[it.key, it.value]}
-          def inputPaths = processedState.collectMany{it.inputPath}
-          def outputFilenames = processedState.collectMany{it.outputFilename}
           
           // convert state to yaml blob
           def yamlBlob_ = toTaggedYamlBlob([id: id_] + updatedState_)
 
-          [id_, yamlBlob_, yamlFilename, inputPaths, outputFilenames]
+          [id_, yamlBlob_, yamlFilename]
         }
         | publishStatesProc
     emit: input_ch
@@ -2559,7 +2687,8 @@ def _debug(workflowArgs, debugKey) {
 def workflowFactory(Map args, Map defaultWfArgs, Map meta) {
   def workflowArgs = processWorkflowArgs(args, defaultWfArgs, meta)
   def key_ = workflowArgs["key"]
-  
+  def multipleArgs = meta.config.allArguments.findAll{ it.multiple }.collect{it.plainName}
+
   workflow workflowInstance {
     take: input_
 
@@ -2716,12 +2845,36 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) {
       }
 
     // TODO: move some of the _meta.join_id wrangling to the safeJoin() function.
-    def chInitialOutput = chArgsWithDefaults
+    def chInitialOutputMulti = chArgsWithDefaults
       | _debug(workflowArgs, "processed")
       // run workflow
       | innerWorkflowFactory(workflowArgs)
-      // check output tuple
+    def chInitialOutputList = chInitialOutputMulti instanceof List ? chInitialOutputMulti : [chInitialOutputMulti]
-      | map { id_, output_ ->
+    assert chInitialOutputList.size() > 0: "should have emitted at least one output channel"
+    // Add a channel ID to the events, which designates the channel the event was emitted from as a running number
+    // This number is used to sort the events later when the events are gathered from across the channels.
+    def chInitialOutputListWithIndexedEvents = chInitialOutputList.withIndex().collect{channel, channelIndex ->
+      def newChannel = channel
+        | map {tuple ->
+          assert tuple instanceof List : 
+          "Error in module '${key_}': element in output channel should be a tuple [id, data, ...otherargs...]\n" +
+          "  Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" +
+          "  Expected class: List. Found: tuple.getClass() is ${tuple.getClass()}"
+        
+          def newEvent = [channelIndex] + tuple
+          return newEvent
+        }
+      return newChannel
+    }
+    // Put the events into 1 channel, cover case where there is only one channel is emitted
+    def chInitialOutput = chInitialOutputList.size() > 1 ? \
+      chInitialOutputListWithIndexedEvents[0].mix(*chInitialOutputListWithIndexedEvents.tail()) : \
+      chInitialOutputListWithIndexedEvents[0]
+    def chInitialOutputProcessed = chInitialOutput
+      | map { tuple  ->
+        def channelId = tuple[0]
+        def id_ = tuple[1]
+        def output_ = tuple[2]
 
         // see if output map contains metadata
         def meta_ =
@@ -2734,19 +2887,94 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) {
         output_ = output_.findAll{k, v -> k != "_meta"}
 
         // check value types
-        output_ = _processOutputValues(output_, meta.config, id_, key_)
+        output_ = _checkValidOutputArgument(output_, meta.config, id_, key_)
 
-        // simplify output if need be
+        [join_id, channelId, id_, output_]
-        if (workflowArgs.auto.simplifyOutput && output_.size() == 1) {
-          output_ = output_.values()[0]
-        }
-
-        [join_id, id_, output_]
       }
       // | view{"chInitialOutput: ${it.take(3)}"}
 
+    // join the output [prev_id, channel_id, new_id, output] with the previous state [prev_id, state, ...]
+    def chPublishWithPreviousState = safeJoin(chInitialOutputProcessed, chRunFiltered, key_)
+      // input tuple format: [join_id, channel_id, id, output, prev_state, ...]
+      // output tuple format: [join_id, channel_id, id, new_state, ...]
+      | map{ tup ->
+        def new_state = workflowArgs.toState(tup.drop(2).take(3))
+        tup.take(3) + [new_state] + tup.drop(5)
+      }
+    if (workflowArgs.auto.publish == "state") {
+      def chPublishFiles = chPublishWithPreviousState
+        // input tuple format: [join_id, channel_id, id, new_state, ...]
+        // output tuple format: [join_id, channel_id, id, new_state]
+        | map{ tup ->
+          tup.take(4)
+        }
+
+      safeJoin(chPublishFiles, chArgsWithDefaults, key_)
+        // input tuple format: [join_id, channel_id, id, new_state, orig_state, ...]
+        // output tuple format: [id, new_state, orig_state]
+        | map { tup ->
+          tup.drop(2).take(3)
+        }
+        | publishFilesByConfig(key: key_, config: meta.config)
+    }
+    // Join the state from the events that were emitted from different channels
+    def chJoined = chInitialOutputProcessed
+      | map {tuple ->
+        def join_id = tuple[0]
+        def channel_id = tuple[1]
+        def id = tuple[2]
+        def other = tuple.drop(3)
+        // Below, groupTuple is used to join the events. To make sure resuming a workflow
+        // keeps working, the output state must be deterministic. This means the state needs to be
+        // sorted with groupTuple's has a 'sort' argument. This argument can be set to 'hash',
+        // but hashing the state when it is large can be problematic in terms of performance.
+        // Therefore, a custom comparator function is provided. We add the channel ID to the 
+        // states so that we can use the channel ID to sort the items. 
+        def stateWithChannelID = [[channel_id] * other.size(), other].transpose()
+        // A comparator that is provided to groupTuple's 'sort' argument is applied
+        // to all elements of the event tuple (that is not the 'id'). The comparator
+        // closure that is used below expects the input to be List. So the join_id and
+        // channel_id must also be wrapped in a list. 
+        [[join_id], [channel_id], id] + stateWithChannelID
+      }
+      | groupTuple(by: 2, sort: {a, b -> a[0] <=> b[0]}, size: chInitialOutputList.size(), remainder: true)
+      | map {join_ids, _, id, statesWithChannelID ->
+        // Remove the channel IDs from the states
+        def states = statesWithChannelID.collect{it[1]}
+        def newJoinId = join_ids.flatten().unique{a, b -> a <=> b}
+        assert newJoinId.size() == 1: "Multiple events were emitted for '$id'."
+        def newJoinIdUnique = newJoinId[0]
+        
+        // Merge the states from the different channels
+        def newState = states.inject([:]){ old_state, state_to_add ->
+          return old_state + state_to_add.collectEntries{k, v -> 
+            if (!multipleArgs.contains(k)) {
+              // if the key is not a multiple argument, we expect only one value
+              if (old_state.containsKey(k)) {
+                assert old_state[k] == v : "ID $id: multiple entries for argument $k were emitted."
+              }
+              [k, v]
+            } else {
+              // if the key is a multiple argument, append the different values into one list
+              def prevValue = old_state.getOrDefault(k, [])
+              def prevValueAsList = prevValue instanceof List ? prevValue : [prevValue]
+              [k, prevValueAsList + v]
+            }
+          }
+        }
+
+        _checkAllRequiredOuputsPresent(newState, meta.config, id, key_)
+
+        // simplify output if need be
+        if (workflowArgs.auto.simplifyOutput && newState.size() == 1) {
+          newState = newState.values()[0]
+        }
+
+        return [newJoinIdUnique, id, newState]
+      }
+    
     // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...]
-    def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_)
+    def chNewState = safeJoin(chJoined, chRunFiltered, key_)
       // input tuple format: [join_id, id, output, prev_state, ...]
       // output tuple format: [join_id, id, new_state, ...]
       | map{ tup ->
@@ -2755,23 +2983,21 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) {
       }
 
     if (workflowArgs.auto.publish == "state") {
-      def chPublish = chNewState
+      def chPublishStates = chNewState
         // input tuple format: [join_id, id, new_state, ...]
         // output tuple format: [join_id, id, new_state]
         | map{ tup ->
           tup.take(3)
         }
 
-      safeJoin(chPublish, chArgsWithDefaults, key_)
+      safeJoin(chPublishStates, chArgsWithDefaults, key_)
         // input tuple format: [join_id, id, new_state, orig_state, ...]
         // output tuple format: [id, new_state, orig_state]
         | map { tup ->
           tup.drop(1).take(3)
-      }
+        }
         | publishStatesByConfig(key: key_, config: meta.config)
     }
-
-    // remove join_id and meta
     chReturn = chNewState
       | map { tup ->
         // input tuple format: [join_id, id, new_state, ...]
@@ -2806,7 +3032,7 @@ meta = [
   "config": processConfig(readJsonBlob('''{
   "name" : "interop_summary_to_csv",
   "namespace" : "io",
-  "version" : "v0.3.7",
+  "version" : "v0.3.9",
   "argument_groups" : [
     {
       "name" : "Input arguments",
@@ -2863,8 +3089,14 @@ meta = [
     }
   ],
   "status" : "enabled",
+  "scope" : {
+    "image" : "public",
+    "target" : "public"
+  },
   "requirements" : {
     "commands" : [
+      "summary",
+      "index-summary",
       "ps"
     ]
   },
@@ -2955,7 +3187,7 @@ meta = [
       "id" : "docker",
       "image" : "debian:stable-slim",
       "target_registry" : "images.viash-hub.com",
-      "target_tag" : "v0.3.7",
+      "target_tag" : "v0.3.9",
       "namespace_separator" : "/",
       "setup" : [
         {
@@ -2984,14 +3216,14 @@ meta = [
     "runner" : "nextflow",
     "engine" : "docker|native",
     "output" : "target/nextflow/io/interop_summary_to_csv",
-    "viash_version" : "0.9.0",
+    "viash_version" : "0.9.4",
-    "git_commit" : "e177226c52ca1dd5a5cf9bb0bfc108812e2aa332",
+    "git_commit" : "820318c378d8119e2aa98768282e43c2aa017ba7",
     "git_remote" : "https://github.com/viash-hub/demultiplex",
-    "git_tag" : "v0.3.5-6-ge177226"
+    "git_tag" : "v0.3.8-5-g820318c"
   },
   "package_config" : {
     "name" : "demultiplex",
-    "version" : "v0.3.7",
+    "version" : "v0.3.9",
     "description" : "Demultiplexing pipeline\n",
     "info" : {
       "test_resources" : [
@@ -3001,14 +3233,14 @@ meta = [
         }
       ]
     },
-    "viash_version" : "0.9.0",
+    "viash_version" : "0.9.4",
     "source" : "src",
     "target" : "target",
     "config_mods" : [
-      ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n",
+      ".requirements.commands += ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n",
       ".engines += { type: \\"native\\" }",
       ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'",
-      ".engines[.type == 'docker'].target_tag := 'v0.3.7'"
+      ".engines[.type == 'docker'].target_tag := 'v0.3.9'"
     ],
     "keywords" : [
       "bioinformatics",
@@ -3404,7 +3636,7 @@ def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) {
   // create process from temp file
   def binding = new nextflow.script.ScriptBinding([:])
   def session = nextflow.Nextflow.getSession()
-  def parser = new nextflow.script.ScriptParser(session)
+  def parser = _getScriptLoader(session)
     .setModule(true)
     .setBinding(binding)
   def moduleScript = parser.runScript(tempFile)
@@ -3418,6 +3650,27 @@ def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) {
   return scriptMeta.getProcess(procKey)
 }
 
+// use Reflection to get a ScriptParser / ScriptLoader
+//   <25.02.0-edge: new nextflow.script.ScriptParser(session)
+//   >=25.02.0-edge: nextflow.script.ScriptLoaderFactory.create(session)
+def _getScriptLoader(nextflow.Session session) {
+  // try using the old method
+  try {
+    Class<?> scriptParserClass = Class.forName('nextflow.script.ScriptParser')
+    return scriptParserClass.getDeclaredConstructor(nextflow.Session).newInstance(session)
+  } catch (ClassNotFoundException e) {
+    // else try with the new method
+    try {
+      Class<?> scriptLoaderFactoryClass = Class.forName('nextflow.script.ScriptLoaderFactory')
+      def createMethod = scriptLoaderFactoryClass.getDeclaredMethod('create', nextflow.Session)
+      return createMethod.invoke(null, session) // null because create is static
+    } catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException | java.lang.reflect.InvocationTargetException e2) {
+      // Handle the case where neither class is found
+      throw new Exception("Neither nextflow.script.ScriptParser nor nextflow.script.ScriptLoaderFactory could be found. Is this a compatible Nextflow version?", e2)
+    }
+  }
+}
+
 // defaults
 meta["defaults"] = [
   // key to be used to trace the process and determine output names
@@ -3431,7 +3684,7 @@ meta["defaults"] = [
   "container" : {
     "registry" : "images.viash-hub.com",
     "image" : "vsh/demultiplex/io/interop_summary_to_csv",
-    "tag" : "v0.3.7"
+    "tag" : "v0.3.9"
   },
   "tag" : "$id"
 }'''),

target/nextflow/io/interop_summary_to_csv/nextflow.config

@@ -2,7 +2,7 @@ manifest {
   name = 'io/interop_summary_to_csv'
   mainScript = 'main.nf'
   nextflowVersion = '!>=20.12.1-edge'
-  version = 'v0.3.7'
+  version = 'v0.3.9'
 }
 
 process.container = 'nextflow/bash:latest'

target/nextflow/io/interop_summary_to_csv/nextflow_schema.json

@@ -37,10 +37,10 @@
                 "output_run_summary": {
                 "type":
                 "string",
-                "description": "Type: `file`, required, default: `$id.$key.output_run_summary.output_run_summary`. ",
+                "description": "Type: `file`, required, default: `$id.$key.output_run_summary`. ",
-                "help_text": "Type: `file`, required, default: `$id.$key.output_run_summary.output_run_summary`. "
+                "help_text": "Type: `file`, required, default: `$id.$key.output_run_summary`. "
             ,
-                "default":"$id.$key.output_run_summary.output_run_summary"
+                "default":"$id.$key.output_run_summary"
             }
     
 
@@ -48,10 +48,10 @@
                 "output_index_summary": {
                 "type":
                 "string",
-                "description": "Type: `file`, required, default: `$id.$key.output_index_summary.output_index_summary`. ",
+                "description": "Type: `file`, required, default: `$id.$key.output_index_summary`. ",
-                "help_text": "Type: `file`, required, default: `$id.$key.output_index_summary.output_index_summary`. "
+                "help_text": "Type: `file`, required, default: `$id.$key.output_index_summary`. "
             ,
-                "default":"$id.$key.output_index_summary.output_index_summary"
+                "default":"$id.$key.output_index_summary"
             }
     
 

target/nextflow/io/publish/.config.vsh.yaml

@@ -1,6 +1,6 @@
 name: "publish"
 namespace: "io"
-version: "v0.3.7"
+version: "v0.3.9"
 argument_groups:
 - name: "Input arguments"
   arguments:
@@ -100,6 +100,9 @@ resources:
 description: "Publish the processed results of the run"
 info: null
 status: "enabled"
+scope:
+  image: "public"
+  target: "public"
 requirements:
   commands:
   - "ps"
@@ -178,7 +181,7 @@ engines:
   id: "docker"
   image: "debian:stable-slim"
   target_registry: "images.viash-hub.com"
-  target_tag: "v0.3.7"
+  target_tag: "v0.3.9"
   namespace_separator: "/"
   setup:
   - type: "apt"
@@ -195,29 +198,29 @@ build_info:
   engine: "docker|native"
   output: "target/nextflow/io/publish"
   executable: "target/nextflow/io/publish/main.nf"
-  viash_version: "0.9.0"
+  viash_version: "0.9.4"
-  git_commit: "e177226c52ca1dd5a5cf9bb0bfc108812e2aa332"
+  git_commit: "820318c378d8119e2aa98768282e43c2aa017ba7"
   git_remote: "https://github.com/viash-hub/demultiplex"
-  git_tag: "v0.3.5-6-ge177226"
+  git_tag: "v0.3.8-5-g820318c"
 package_config:
   name: "demultiplex"
-  version: "v0.3.7"
+  version: "v0.3.9"
   description: "Demultiplexing pipeline\n"
   info:
     test_resources:
     - path: "gs://viash-hub-test-data/demultiplex/v2/"
       dest: "testData"
-  viash_version: "0.9.0"
+  viash_version: "0.9.4"
   source: "src"
   target: "target"
   config_mods:
-  - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\
+  - ".requirements.commands += ['ps']\n.runners[.type == 'nextflow'].directives.tag\
     \ := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n\
     .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\
     )'\n"
   - ".engines += { type: \"native\" }"
   - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
-  - ".engines[.type == 'docker'].target_tag := 'v0.3.7'"
+  - ".engines[.type == 'docker'].target_tag := 'v0.3.9'"
   keywords:
   - "bioinformatics"
   - "sequence"

target/nextflow/io/publish/main.nf

@@ -1,6 +1,6 @@
-// publish v0.3.7
+// publish v0.3.9
 // 
-// This wrapper script is auto-generated by viash 0.9.0 and is thus a derivative
+// This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative
 // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data
 // Intuitive.
 // 
@@ -82,64 +82,56 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi
       foundClass = "List[${e.foundClass}]"
     }
   } else if (par.type == "string") {
-    // cast to string if need be
+    // cast to string if need be. only cast if the value is a GString
     if (value instanceof GString) {
-      value = value.toString()
+      value = value as String
     }
     expectedClass = value instanceof String ? null : "String"
   } else if (par.type == "integer") {
     // cast to integer if need be
-    if (value instanceof String) {
+    if (value !instanceof Integer) {
       try {
-        value = value.toInteger()
+        value = value as Integer
       } catch (NumberFormatException e) {
-        // do nothing
+        expectedClass = "Integer"
       }
     }
-    if (value instanceof java.math.BigInteger) {
-      value = value.intValue()
-    }
-    expectedClass = value instanceof Integer ? null : "Integer"
   } else if (par.type == "long") {
     // cast to long if need be
-    if (value instanceof String) {
+    if (value !instanceof Long) {
       try {
-        value = value.toLong()
+        value = value as Long
       } catch (NumberFormatException e) {
-        // do nothing
+        expectedClass = "Long"
       }
     }
-    if (value instanceof Integer) {
-      value = value.toLong()
-    }
-    expectedClass = value instanceof Long ? null : "Long"
   } else if (par.type == "double") {
     // cast to double if need be
-    if (value instanceof String) {
+    if (value !instanceof Double) {
       try {
-        value = value.toDouble()
+        value = value as Double
       } catch (NumberFormatException e) {
-        // do nothing
+        expectedClass = "Double"
       }
     }
-    if (value instanceof java.math.BigDecimal) {
+  } else if (par.type == "float") {
-      value = value.doubleValue()
+    // cast to float if need be
+    if (value !instanceof Float) {
+      try {
+        value = value as Float
+      } catch (NumberFormatException e) {
+        expectedClass = "Float"
+      }
     }
-    if (value instanceof Float) {
-      value = value.toDouble()
-    }
-    expectedClass = value instanceof Double ? null : "Double"
   } else if (par.type == "boolean" | par.type == "boolean_true" | par.type == "boolean_false") {
     // cast to boolean if need be
-    if (value instanceof String) {
+    if (value !instanceof Boolean) {
-      def valueLower = value.toLowerCase()
+      try {
-      if (valueLower == "true") {
+        value = value as Boolean
-        value = true
+      } catch (Exception e) {
-      } else if (valueLower == "false") {
+        expectedClass = "Boolean"
-        value = false
       }
     }
-    expectedClass = value instanceof Boolean ? null : "Boolean"
   } else if (par.type == "file" && (par.direction == "input" || stage == "output")) {
     // cast to path if need be
     if (value instanceof String) {
@@ -151,10 +143,13 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi
     expectedClass = value instanceof Path ? null : "Path"
   } else if (par.type == "file" && stage == "input" && par.direction == "output") {
     // cast to string if need be
-    if (value instanceof GString) {
+    if (value !instanceof String) {
-      value = value.toString()
+      try {
+        value = value as String
+      } catch (Exception e) {
+        expectedClass = "String"
+      }
     }
-    expectedClass = value instanceof String ? null : "String"
   } else {
     // didn't find a match for par.type
     expectedClass = par.type
@@ -173,7 +168,7 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi
 Map _processInputValues(Map inputs, Map config, String id, String key) {
   if (!workflow.stubRun) {
     config.allArguments.each { arg ->
-      if (arg.required) {
+      if (arg.required && arg.direction == "input") {
         assert inputs.containsKey(arg.plainName) && inputs.get(arg.plainName) != null : 
           "Error in module '${key}' id '${id}': required input argument '${arg.plainName}' is missing"
       }
@@ -192,15 +187,8 @@ Map _processInputValues(Map inputs, Map config, String id, String key) {
 }
 
 // helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_processOutputValues.nf'
-Map _processOutputValues(Map outputs, Map config, String id, String key) {
+Map _checkValidOutputArgument(Map outputs, Map config, String id, String key) {
   if (!workflow.stubRun) {
-    config.allArguments.each { arg ->
-      if (arg.direction == "output" && arg.required) {
-        assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : 
-          "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing"
-      }
-    }
-
     outputs = outputs.collectEntries { name, value ->
       def par = config.allArguments.find { it.plainName == name && it.direction == "output" }
       assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid output argument"
@@ -213,6 +201,16 @@ Map _processOutputValues(Map outputs, Map config, String id, String key) {
   return outputs
 }
 
+void _checkAllRequiredOuputsPresent(Map outputs, Map config, String id, String key) {
+  if (!workflow.stubRun) {
+    config.allArguments.each { arg ->
+      if (arg.direction == "output" && arg.required) {
+        assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : 
+          "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing"
+      }
+    }
+  }
+}
 // helper file: 'src/main/resources/io/viash/runners/nextflow/channel/IDChecker.nf'
 class IDChecker {
   final def items = [] as Set
@@ -1666,6 +1664,162 @@ def joinStates(Closure apply_) {
   }
   return joinStatesWf
 }
+// helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishFiles.nf'
+def publishFiles(Map args) {
+  def key_ = args.get("key")
+
+  assert key_ != null : "publishFiles: key must be specified"
+  
+  workflow publishFilesWf {
+    take: input_ch
+    main:
+      input_ch
+        | map { tup ->
+          def id_ = tup[0]
+          def state_ = tup[1]
+
+          // the input files and the target output filenames
+          def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose()
+          def inputFiles_ = inputoutputFilenames_[0]
+          def outputFilenames_ = inputoutputFilenames_[1]
+
+          [id_, inputFiles_, outputFilenames_]
+        }
+        | publishFilesProc
+    emit: input_ch
+  }
+  return publishFilesWf
+}
+
+process publishFilesProc {
+  // todo: check publishpath?
+  publishDir path: "${getPublishDir()}/", mode: "copy"
+  tag "$id"
+  input:
+    tuple val(id), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles)
+  output:
+    tuple val(id), path{outputFiles}
+  script:
+  def copyCommands = [
+    inputFiles instanceof List ? inputFiles : [inputFiles],
+    outputFiles instanceof List ? outputFiles : [outputFiles]
+  ]
+    .transpose()
+    .collectMany{infile, outfile ->
+      if (infile.toString() != outfile.toString()) {
+        [
+          "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"",
+          "cp -r '${infile.toString()}' '${outfile.toString()}'"
+        ]
+      } else {
+        // no need to copy if infile is the same as outfile
+        []
+      }
+    }
+  """
+  echo "Copying output files to destination folder"
+  ${copyCommands.join("\n  ")}
+  """
+}
+
+
+// this assumes that the state contains no other values other than those specified in the config
+def publishFilesByConfig(Map args) {
+  def config = args.get("config")
+  assert config != null : "publishFilesByConfig: config must be specified"
+
+  def key_ = args.get("key", config.name)
+  assert key_ != null : "publishFilesByConfig: key must be specified"
+  
+  workflow publishFilesSimpleWf {
+    take: input_ch
+    main:
+      input_ch
+        | map { tup ->
+          def id_ = tup[0]
+          def state_ = tup[1] // e.g. [output: new File("myoutput.h5ad"), k: 10]
+          def origState_ = tup[2] // e.g. [output: '$id.$key.foo.h5ad']
+
+
+          // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where
+          //   - key is a String
+          //   - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path)
+          //   - inputPath is a List[Path]
+          //   - outputFilename is a List[String]
+          //   - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml)
+          def processedState =
+            config.allArguments
+              .findAll { it.direction == "output" }
+              .collectMany { par ->
+                def plainName_ = par.plainName
+                // if the state does not contain the key, it's an
+                // optional argument for which the component did 
+                // not generate any output OR multiple channels were emitted
+                // and the output was just not added to using the channel
+                // that is now being parsed
+                if (!state_.containsKey(plainName_)) {
+                  return []
+                }
+                def value = state_[plainName_]
+                // if the parameter is not a file, it should be stored
+                // in the state as-is, but is not something that needs 
+                // to be copied from the source path to the dest path
+                if (par.type != "file") {
+                  return [[inputPath: [], outputFilename: []]]
+                }
+                // if the orig state does not contain this filename,
+                // it's an optional argument for which the user specified
+                // that it should not be returned as a state
+                if (!origState_.containsKey(plainName_)) {
+                  return []
+                }
+                def filenameTemplate = origState_[plainName_]
+                // if the pararameter is multiple: true, fetch the template
+                if (par.multiple && filenameTemplate instanceof List) {
+                  filenameTemplate = filenameTemplate[0]
+                }
+                // instantiate the template
+                def filename = filenameTemplate
+                  .replaceAll('\\$id', id_)
+                  .replaceAll('\\$\\{id\\}', id_)
+                  .replaceAll('\\$key', key_)
+                  .replaceAll('\\$\\{key\\}', key_)
+                if (par.multiple) {
+                  // if the parameter is multiple: true, the filename
+                  // should contain a wildcard '*' that is replaced with
+                  // the index of the file
+                  assert filename.contains("*") : "Module '${key_}' id '${id_}': Multiple output files specified, but no wildcard '*' in the filename: ${filename}"
+                  def outputPerFile = value.withIndex().collect{ val, ix ->
+                    def filename_ix = filename.replace("*", ix.toString())
+                    def inputPath = val instanceof File ? val.toPath() : val
+                    [inputPath: inputPath, outputFilename: filename_ix]
+                  }
+                  def transposedOutputs = ["inputPath", "outputFilename"].collectEntries{ key -> 
+                    [key, outputPerFile.collect{dic -> dic[key]}]
+                  }
+                  return [[key: plainName_] + transposedOutputs]
+                } else {
+                  def value_ = java.nio.file.Paths.get(filename)
+                  def inputPath = value instanceof File ? value.toPath() : value
+                  return [[inputPath: [inputPath], outputFilename: [filename]]]
+                }
+              }
+          
+          def inputPaths = processedState.collectMany{it.inputPath}
+          def outputFilenames = processedState.collectMany{it.outputFilename}
+          
+
+          [id_, inputPaths, outputFilenames]
+        }
+        | publishFilesProc
+    emit: input_ch
+  }
+  return publishFilesSimpleWf
+}
+
+
+
+
 // helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishStates.nf'
 def collectFiles(obj) {
   if (obj instanceof java.io.File || obj instanceof Path)  {
@@ -1723,8 +1877,6 @@ def publishStates(Map args) {
 
           // the input files and the target output filenames
           def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose()
-          def inputFiles_ = inputoutputFilenames_[0]
-          def outputFilenames_ = inputoutputFilenames_[1]
 
           def yamlFilename = yamlTemplate_
             .replaceAll('\\$id', id_)
@@ -1737,7 +1889,7 @@ def publishStates(Map args) {
           // convert state to yaml blob
           def yamlBlob_ = toRelativeTaggedYamlBlob([id: id_] + state_, java.nio.file.Paths.get(yamlFilename))
 
-          [id_, yamlBlob_, yamlFilename, inputFiles_, outputFilenames_]
+          [id_, yamlBlob_, yamlFilename]
         }
         | publishStatesProc
     emit: input_ch
@@ -1749,33 +1901,17 @@ process publishStatesProc {
   publishDir path: "${getPublishDir()}/", mode: "copy"
   tag "$id"
   input:
-    tuple val(id), val(yamlBlob), val(yamlFile), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles)
+    tuple val(id), val(yamlBlob), val(yamlFile)
   output:
-    tuple val(id), path{[yamlFile] + outputFiles}
+    tuple val(id), path{[yamlFile]}
   script:
-  def copyCommands = [
-    inputFiles instanceof List ? inputFiles : [inputFiles],
-    outputFiles instanceof List ? outputFiles : [outputFiles]
-  ]
-    .transpose()
-    .collectMany{infile, outfile ->
-      if (infile.toString() != outfile.toString()) {
-        [
-          "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"",
-          "cp -r '${infile.toString()}' '${outfile.toString()}'"
-        ]
-      } else {
-        // no need to copy if infile is the same as outfile
-        []
-      }
-    }
   """
-mkdir -p "\$(dirname '${yamlFile}')"
+  mkdir -p "\$(dirname '${yamlFile}')"
-echo "Storing state as yaml"
+  echo "Storing state as yaml"
-echo '${yamlBlob}' > '${yamlFile}'
+  cat > '${yamlFile}' << HERE
-echo "Copying output files to destination folder"
+${yamlBlob}
-${copyCommands.join("\n  ")}
+HERE
-"""
+  """
 }
 
 
@@ -1806,13 +1942,10 @@ def publishStatesByConfig(Map args) {
             .replaceAll('\\$\\{key\\}', key_)
           def yamlDir = java.nio.file.Paths.get(yamlFilename).getParent()
 
-          // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where
+          // the processed state is a list of [key, value] tuples, where
           //   - key is a String
           //   - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path)
-          //   - inputPath is a List[Path]
-          //   - outputFilename is a List[String]
           //   - (key, value) are the tuples that will be saved to the state.yaml file
-          //   - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml)
           def processedState =
             config.allArguments
               .findAll { it.direction == "output" }
@@ -1829,7 +1962,7 @@ def publishStatesByConfig(Map args) {
                 // in the state as-is, but is not something that needs 
                 // to be copied from the source path to the dest path
                 if (par.type != "file") {
-                  return [[key: plainName_, value: value, inputPath: [], outputFilename: []]]
+                  return [[key: plainName_, value: value]]
                 }
                 // if the orig state does not contain this filename,
                 // it's an optional argument for which the user specified
@@ -1860,13 +1993,9 @@ def publishStatesByConfig(Map args) {
                     if (yamlDir != null) {
                       value_ = yamlDir.relativize(value_)
                     }
-                    def inputPath = val instanceof File ? val.toPath() : val
+                    return value_
-                    [value: value_, inputPath: inputPath, outputFilename: filename_ix]
                   }
-                  def transposedOutputs = ["value", "inputPath", "outputFilename"].collectEntries{ key -> 
+                  return [["key": plainName_, "value": outputPerFile]]
-                    [key, outputPerFile.collect{dic -> dic[key]}]
-                  }
-                  return [[key: plainName_] + transposedOutputs]
                 } else {
                   def value_ = java.nio.file.Paths.get(filename)
                   // if id contains a slash
@@ -1874,18 +2003,17 @@ def publishStatesByConfig(Map args) {
                     value_ = yamlDir.relativize(value_)
                   }
                   def inputPath = value instanceof File ? value.toPath() : value
-                  return [[key: plainName_, value: value_, inputPath: [inputPath], outputFilename: [filename]]]
+                  return [["key": plainName_, value: value_]]
                 }
               }
+              
           
           def updatedState_ = processedState.collectEntries{[it.key, it.value]}
-          def inputPaths = processedState.collectMany{it.inputPath}
-          def outputFilenames = processedState.collectMany{it.outputFilename}
           
           // convert state to yaml blob
           def yamlBlob_ = toTaggedYamlBlob([id: id_] + updatedState_)
 
-          [id_, yamlBlob_, yamlFilename, inputPaths, outputFilenames]
+          [id_, yamlBlob_, yamlFilename]
         }
         | publishStatesProc
     emit: input_ch
@@ -2559,7 +2687,8 @@ def _debug(workflowArgs, debugKey) {
 def workflowFactory(Map args, Map defaultWfArgs, Map meta) {
   def workflowArgs = processWorkflowArgs(args, defaultWfArgs, meta)
   def key_ = workflowArgs["key"]
-  
+  def multipleArgs = meta.config.allArguments.findAll{ it.multiple }.collect{it.plainName}
+
   workflow workflowInstance {
     take: input_
 
@@ -2716,12 +2845,36 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) {
       }
 
     // TODO: move some of the _meta.join_id wrangling to the safeJoin() function.
-    def chInitialOutput = chArgsWithDefaults
+    def chInitialOutputMulti = chArgsWithDefaults
       | _debug(workflowArgs, "processed")
       // run workflow
       | innerWorkflowFactory(workflowArgs)
-      // check output tuple
+    def chInitialOutputList = chInitialOutputMulti instanceof List ? chInitialOutputMulti : [chInitialOutputMulti]
-      | map { id_, output_ ->
+    assert chInitialOutputList.size() > 0: "should have emitted at least one output channel"
+    // Add a channel ID to the events, which designates the channel the event was emitted from as a running number
+    // This number is used to sort the events later when the events are gathered from across the channels.
+    def chInitialOutputListWithIndexedEvents = chInitialOutputList.withIndex().collect{channel, channelIndex ->
+      def newChannel = channel
+        | map {tuple ->
+          assert tuple instanceof List : 
+          "Error in module '${key_}': element in output channel should be a tuple [id, data, ...otherargs...]\n" +
+          "  Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" +
+          "  Expected class: List. Found: tuple.getClass() is ${tuple.getClass()}"
+        
+          def newEvent = [channelIndex] + tuple
+          return newEvent
+        }
+      return newChannel
+    }
+    // Put the events into 1 channel, cover case where there is only one channel is emitted
+    def chInitialOutput = chInitialOutputList.size() > 1 ? \
+      chInitialOutputListWithIndexedEvents[0].mix(*chInitialOutputListWithIndexedEvents.tail()) : \
+      chInitialOutputListWithIndexedEvents[0]
+    def chInitialOutputProcessed = chInitialOutput
+      | map { tuple  ->
+        def channelId = tuple[0]
+        def id_ = tuple[1]
+        def output_ = tuple[2]
 
         // see if output map contains metadata
         def meta_ =
@@ -2734,19 +2887,94 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) {
         output_ = output_.findAll{k, v -> k != "_meta"}
 
         // check value types
-        output_ = _processOutputValues(output_, meta.config, id_, key_)
+        output_ = _checkValidOutputArgument(output_, meta.config, id_, key_)
 
-        // simplify output if need be
+        [join_id, channelId, id_, output_]
-        if (workflowArgs.auto.simplifyOutput && output_.size() == 1) {
-          output_ = output_.values()[0]
-        }
-
-        [join_id, id_, output_]
       }
       // | view{"chInitialOutput: ${it.take(3)}"}
 
+    // join the output [prev_id, channel_id, new_id, output] with the previous state [prev_id, state, ...]
+    def chPublishWithPreviousState = safeJoin(chInitialOutputProcessed, chRunFiltered, key_)
+      // input tuple format: [join_id, channel_id, id, output, prev_state, ...]
+      // output tuple format: [join_id, channel_id, id, new_state, ...]
+      | map{ tup ->
+        def new_state = workflowArgs.toState(tup.drop(2).take(3))
+        tup.take(3) + [new_state] + tup.drop(5)
+      }
+    if (workflowArgs.auto.publish == "state") {
+      def chPublishFiles = chPublishWithPreviousState
+        // input tuple format: [join_id, channel_id, id, new_state, ...]
+        // output tuple format: [join_id, channel_id, id, new_state]
+        | map{ tup ->
+          tup.take(4)
+        }
+
+      safeJoin(chPublishFiles, chArgsWithDefaults, key_)
+        // input tuple format: [join_id, channel_id, id, new_state, orig_state, ...]
+        // output tuple format: [id, new_state, orig_state]
+        | map { tup ->
+          tup.drop(2).take(3)
+        }
+        | publishFilesByConfig(key: key_, config: meta.config)
+    }
+    // Join the state from the events that were emitted from different channels
+    def chJoined = chInitialOutputProcessed
+      | map {tuple ->
+        def join_id = tuple[0]
+        def channel_id = tuple[1]
+        def id = tuple[2]
+        def other = tuple.drop(3)
+        // Below, groupTuple is used to join the events. To make sure resuming a workflow
+        // keeps working, the output state must be deterministic. This means the state needs to be
+        // sorted with groupTuple's has a 'sort' argument. This argument can be set to 'hash',
+        // but hashing the state when it is large can be problematic in terms of performance.
+        // Therefore, a custom comparator function is provided. We add the channel ID to the 
+        // states so that we can use the channel ID to sort the items. 
+        def stateWithChannelID = [[channel_id] * other.size(), other].transpose()
+        // A comparator that is provided to groupTuple's 'sort' argument is applied
+        // to all elements of the event tuple (that is not the 'id'). The comparator
+        // closure that is used below expects the input to be List. So the join_id and
+        // channel_id must also be wrapped in a list. 
+        [[join_id], [channel_id], id] + stateWithChannelID
+      }
+      | groupTuple(by: 2, sort: {a, b -> a[0] <=> b[0]}, size: chInitialOutputList.size(), remainder: true)
+      | map {join_ids, _, id, statesWithChannelID ->
+        // Remove the channel IDs from the states
+        def states = statesWithChannelID.collect{it[1]}
+        def newJoinId = join_ids.flatten().unique{a, b -> a <=> b}
+        assert newJoinId.size() == 1: "Multiple events were emitted for '$id'."
+        def newJoinIdUnique = newJoinId[0]
+        
+        // Merge the states from the different channels
+        def newState = states.inject([:]){ old_state, state_to_add ->
+          return old_state + state_to_add.collectEntries{k, v -> 
+            if (!multipleArgs.contains(k)) {
+              // if the key is not a multiple argument, we expect only one value
+              if (old_state.containsKey(k)) {
+                assert old_state[k] == v : "ID $id: multiple entries for argument $k were emitted."
+              }
+              [k, v]
+            } else {
+              // if the key is a multiple argument, append the different values into one list
+              def prevValue = old_state.getOrDefault(k, [])
+              def prevValueAsList = prevValue instanceof List ? prevValue : [prevValue]
+              [k, prevValueAsList + v]
+            }
+          }
+        }
+
+        _checkAllRequiredOuputsPresent(newState, meta.config, id, key_)
+
+        // simplify output if need be
+        if (workflowArgs.auto.simplifyOutput && newState.size() == 1) {
+          newState = newState.values()[0]
+        }
+
+        return [newJoinIdUnique, id, newState]
+      }
+    
     // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...]
-    def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_)
+    def chNewState = safeJoin(chJoined, chRunFiltered, key_)
       // input tuple format: [join_id, id, output, prev_state, ...]
       // output tuple format: [join_id, id, new_state, ...]
       | map{ tup ->
@@ -2755,23 +2983,21 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) {
       }
 
     if (workflowArgs.auto.publish == "state") {
-      def chPublish = chNewState
+      def chPublishStates = chNewState
         // input tuple format: [join_id, id, new_state, ...]
         // output tuple format: [join_id, id, new_state]
         | map{ tup ->
           tup.take(3)
         }
 
-      safeJoin(chPublish, chArgsWithDefaults, key_)
+      safeJoin(chPublishStates, chArgsWithDefaults, key_)
         // input tuple format: [join_id, id, new_state, orig_state, ...]
         // output tuple format: [id, new_state, orig_state]
         | map { tup ->
           tup.drop(1).take(3)
-      }
+        }
         | publishStatesByConfig(key: key_, config: meta.config)
     }
-
-    // remove join_id and meta
     chReturn = chNewState
       | map { tup ->
         // input tuple format: [join_id, id, new_state, ...]
@@ -2806,7 +3032,7 @@ meta = [
   "config": processConfig(readJsonBlob('''{
   "name" : "publish",
   "namespace" : "io",
-  "version" : "v0.3.7",
+  "version" : "v0.3.9",
   "argument_groups" : [
     {
       "name" : "Input arguments",
@@ -2929,6 +3155,10 @@ meta = [
   ],
   "description" : "Publish the processed results of the run",
   "status" : "enabled",
+  "scope" : {
+    "image" : "public",
+    "target" : "public"
+  },
   "requirements" : {
     "commands" : [
       "ps"
@@ -3021,7 +3251,7 @@ meta = [
       "id" : "docker",
       "image" : "debian:stable-slim",
       "target_registry" : "images.viash-hub.com",
-      "target_tag" : "v0.3.7",
+      "target_tag" : "v0.3.9",
       "namespace_separator" : "/",
       "setup" : [
         {
@@ -3043,14 +3273,14 @@ meta = [
     "runner" : "nextflow",
     "engine" : "docker|native",
     "output" : "target/nextflow/io/publish",
-    "viash_version" : "0.9.0",
+    "viash_version" : "0.9.4",
-    "git_commit" : "e177226c52ca1dd5a5cf9bb0bfc108812e2aa332",
+    "git_commit" : "820318c378d8119e2aa98768282e43c2aa017ba7",
     "git_remote" : "https://github.com/viash-hub/demultiplex",
-    "git_tag" : "v0.3.5-6-ge177226"
+    "git_tag" : "v0.3.8-5-g820318c"
   },
   "package_config" : {
     "name" : "demultiplex",
-    "version" : "v0.3.7",
+    "version" : "v0.3.9",
     "description" : "Demultiplexing pipeline\n",
     "info" : {
       "test_resources" : [
@@ -3060,14 +3290,14 @@ meta = [
         }
       ]
     },
-    "viash_version" : "0.9.0",
+    "viash_version" : "0.9.4",
     "source" : "src",
     "target" : "target",
     "config_mods" : [
-      ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n",
+      ".requirements.commands += ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n",
       ".engines += { type: \\"native\\" }",
       ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'",
-      ".engines[.type == 'docker'].target_tag := 'v0.3.7'"
+      ".engines[.type == 'docker'].target_tag := 'v0.3.9'"
     ],
     "keywords" : [
       "bioinformatics",
@@ -3491,7 +3721,7 @@ def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) {
   // create process from temp file
   def binding = new nextflow.script.ScriptBinding([:])
   def session = nextflow.Nextflow.getSession()
-  def parser = new nextflow.script.ScriptParser(session)
+  def parser = _getScriptLoader(session)
     .setModule(true)
     .setBinding(binding)
   def moduleScript = parser.runScript(tempFile)
@@ -3505,6 +3735,27 @@ def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) {
   return scriptMeta.getProcess(procKey)
 }
 
+// use Reflection to get a ScriptParser / ScriptLoader
+//   <25.02.0-edge: new nextflow.script.ScriptParser(session)
+//   >=25.02.0-edge: nextflow.script.ScriptLoaderFactory.create(session)
+def _getScriptLoader(nextflow.Session session) {
+  // try using the old method
+  try {
+    Class<?> scriptParserClass = Class.forName('nextflow.script.ScriptParser')
+    return scriptParserClass.getDeclaredConstructor(nextflow.Session).newInstance(session)
+  } catch (ClassNotFoundException e) {
+    // else try with the new method
+    try {
+      Class<?> scriptLoaderFactoryClass = Class.forName('nextflow.script.ScriptLoaderFactory')
+      def createMethod = scriptLoaderFactoryClass.getDeclaredMethod('create', nextflow.Session)
+      return createMethod.invoke(null, session) // null because create is static
+    } catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException | java.lang.reflect.InvocationTargetException e2) {
+      // Handle the case where neither class is found
+      throw new Exception("Neither nextflow.script.ScriptParser nor nextflow.script.ScriptLoaderFactory could be found. Is this a compatible Nextflow version?", e2)
+    }
+  }
+}
+
 // defaults
 meta["defaults"] = [
   // key to be used to trace the process and determine output names
@@ -3518,7 +3769,7 @@ meta["defaults"] = [
   "container" : {
     "registry" : "images.viash-hub.com",
     "image" : "vsh/demultiplex/io/publish",
-    "tag" : "v0.3.7"
+    "tag" : "v0.3.9"
   },
   "tag" : "$id"
 }'''),

target/nextflow/io/publish/nextflow.config

@@ -2,7 +2,7 @@ manifest {
   name = 'io/publish'
   mainScript = 'main.nf'
   nextflowVersion = '!>=20.12.1-edge'
-  version = 'v0.3.7'
+  version = 'v0.3.9'
   description = 'Publish the processed results of the run'
 }
 

target/nextflow/io/publish/nextflow_schema.json

@@ -67,10 +67,10 @@
                 "output": {
                 "type":
                 "string",
-                "description": "Type: `file`, default: `$id.$key.output.output`. ",
+                "description": "Type: `file`, default: `fastq`. ",
-                "help_text": "Type: `file`, default: `$id.$key.output.output`. "
+                "help_text": "Type: `file`, default: `fastq`. "
             ,
-                "default":"$id.$key.output.output"
+                "default":"fastq"
             }
     
 
@@ -78,10 +78,10 @@
                 "output_falco": {
                 "type":
                 "string",
-                "description": "Type: `file`, default: `$id.$key.output_falco.output_falco`. ",
+                "description": "Type: `file`, default: `qc/fastqc`. ",
-                "help_text": "Type: `file`, default: `$id.$key.output_falco.output_falco`. "
+                "help_text": "Type: `file`, default: `qc/fastqc`. "
             ,
-                "default":"$id.$key.output_falco.output_falco"
+                "default":"qc/fastqc"
             }
     
 
@@ -89,10 +89,10 @@
                 "output_multiqc": {
                 "type":
                 "string",
-                "description": "Type: `file`, default: `$id.$key.output_multiqc.html`. ",
+                "description": "Type: `file`, default: `qc/multiqc_report.html`. ",
-                "help_text": "Type: `file`, default: `$id.$key.output_multiqc.html`. "
+                "help_text": "Type: `file`, default: `qc/multiqc_report.html`. "
             ,
-                "default":"$id.$key.output_multiqc.html"
+                "default":"qc/multiqc_report.html"
             }
     
 
@@ -100,10 +100,10 @@
                 "output_run_information": {
                 "type":
                 "string",
-                "description": "Type: `file`, default: `$id.$key.output_run_information.csv`. ",
+                "description": "Type: `file`, default: `run_information.csv`. ",
-                "help_text": "Type: `file`, default: `$id.$key.output_run_information.csv`. "
+                "help_text": "Type: `file`, default: `run_information.csv`. "
             ,
-                "default":"$id.$key.output_run_information.csv"
+                "default":"run_information.csv"
             }
     
 

target/nextflow/io/untar/.config.vsh.yaml

@@ -1,6 +1,6 @@
 name: "untar"
 namespace: "io"
-version: "v0.3.7"
+version: "v0.3.9"
 argument_groups:
 - name: "Input arguments"
   arguments:
@@ -57,6 +57,9 @@ test_resources:
   is_executable: true
 info: null
 status: "enabled"
+scope:
+  image: "public"
+  target: "public"
 requirements:
   commands:
   - "ps"
@@ -135,7 +138,7 @@ engines:
   id: "docker"
   image: "debian:stable-slim"
   target_registry: "images.viash-hub.com"
-  target_tag: "v0.3.7"
+  target_tag: "v0.3.9"
   namespace_separator: "/"
   setup:
   - type: "apt"
@@ -152,29 +155,29 @@ build_info:
   engine: "docker|native"
   output: "target/nextflow/io/untar"
   executable: "target/nextflow/io/untar/main.nf"
-  viash_version: "0.9.0"
+  viash_version: "0.9.4"
-  git_commit: "e177226c52ca1dd5a5cf9bb0bfc108812e2aa332"
+  git_commit: "820318c378d8119e2aa98768282e43c2aa017ba7"
   git_remote: "https://github.com/viash-hub/demultiplex"
-  git_tag: "v0.3.5-6-ge177226"
+  git_tag: "v0.3.8-5-g820318c"
 package_config:
   name: "demultiplex"
-  version: "v0.3.7"
+  version: "v0.3.9"
   description: "Demultiplexing pipeline\n"
   info:
     test_resources:
     - path: "gs://viash-hub-test-data/demultiplex/v2/"
       dest: "testData"
-  viash_version: "0.9.0"
+  viash_version: "0.9.4"
   source: "src"
   target: "target"
   config_mods:
-  - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\
+  - ".requirements.commands += ['ps']\n.runners[.type == 'nextflow'].directives.tag\
     \ := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n\
     .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\
     )'\n"
   - ".engines += { type: \"native\" }"
   - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
-  - ".engines[.type == 'docker'].target_tag := 'v0.3.7'"
+  - ".engines[.type == 'docker'].target_tag := 'v0.3.9'"
   keywords:
   - "bioinformatics"
   - "sequence"

target/nextflow/io/untar/main.nf

@@ -1,6 +1,6 @@
-// untar v0.3.7
+// untar v0.3.9
 // 
-// This wrapper script is auto-generated by viash 0.9.0 and is thus a derivative
+// This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative
 // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data
 // Intuitive.
 // 
@@ -82,64 +82,56 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi
       foundClass = "List[${e.foundClass}]"
     }
   } else if (par.type == "string") {
-    // cast to string if need be
+    // cast to string if need be. only cast if the value is a GString
     if (value instanceof GString) {
-      value = value.toString()
+      value = value as String
     }
     expectedClass = value instanceof String ? null : "String"
   } else if (par.type == "integer") {
     // cast to integer if need be
-    if (value instanceof String) {
+    if (value !instanceof Integer) {
       try {
-        value = value.toInteger()
+        value = value as Integer
       } catch (NumberFormatException e) {
-        // do nothing
+        expectedClass = "Integer"
       }
     }
-    if (value instanceof java.math.BigInteger) {
-      value = value.intValue()
-    }
-    expectedClass = value instanceof Integer ? null : "Integer"
   } else if (par.type == "long") {
     // cast to long if need be
-    if (value instanceof String) {
+    if (value !instanceof Long) {
       try {
-        value = value.toLong()
+        value = value as Long
       } catch (NumberFormatException e) {
-        // do nothing
+        expectedClass = "Long"
       }
     }
-    if (value instanceof Integer) {
-      value = value.toLong()
-    }
-    expectedClass = value instanceof Long ? null : "Long"
   } else if (par.type == "double") {
     // cast to double if need be
-    if (value instanceof String) {
+    if (value !instanceof Double) {
       try {
-        value = value.toDouble()
+        value = value as Double
       } catch (NumberFormatException e) {
-        // do nothing
+        expectedClass = "Double"
       }
     }
-    if (value instanceof java.math.BigDecimal) {
+  } else if (par.type == "float") {
-      value = value.doubleValue()
+    // cast to float if need be
+    if (value !instanceof Float) {
+      try {
+        value = value as Float
+      } catch (NumberFormatException e) {
+        expectedClass = "Float"
+      }
     }
-    if (value instanceof Float) {
-      value = value.toDouble()
-    }
-    expectedClass = value instanceof Double ? null : "Double"
   } else if (par.type == "boolean" | par.type == "boolean_true" | par.type == "boolean_false") {
     // cast to boolean if need be
-    if (value instanceof String) {
+    if (value !instanceof Boolean) {
-      def valueLower = value.toLowerCase()
+      try {
-      if (valueLower == "true") {
+        value = value as Boolean
-        value = true
+      } catch (Exception e) {
-      } else if (valueLower == "false") {
+        expectedClass = "Boolean"
-        value = false
       }
     }
-    expectedClass = value instanceof Boolean ? null : "Boolean"
   } else if (par.type == "file" && (par.direction == "input" || stage == "output")) {
     // cast to path if need be
     if (value instanceof String) {
@@ -151,10 +143,13 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi
     expectedClass = value instanceof Path ? null : "Path"
   } else if (par.type == "file" && stage == "input" && par.direction == "output") {
     // cast to string if need be
-    if (value instanceof GString) {
+    if (value !instanceof String) {
-      value = value.toString()
+      try {
+        value = value as String
+      } catch (Exception e) {
+        expectedClass = "String"
+      }
     }
-    expectedClass = value instanceof String ? null : "String"
   } else {
     // didn't find a match for par.type
     expectedClass = par.type
@@ -173,7 +168,7 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi
 Map _processInputValues(Map inputs, Map config, String id, String key) {
   if (!workflow.stubRun) {
     config.allArguments.each { arg ->
-      if (arg.required) {
+      if (arg.required && arg.direction == "input") {
         assert inputs.containsKey(arg.plainName) && inputs.get(arg.plainName) != null : 
           "Error in module '${key}' id '${id}': required input argument '${arg.plainName}' is missing"
       }
@@ -192,15 +187,8 @@ Map _processInputValues(Map inputs, Map config, String id, String key) {
 }
 
 // helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_processOutputValues.nf'
-Map _processOutputValues(Map outputs, Map config, String id, String key) {
+Map _checkValidOutputArgument(Map outputs, Map config, String id, String key) {
   if (!workflow.stubRun) {
-    config.allArguments.each { arg ->
-      if (arg.direction == "output" && arg.required) {
-        assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : 
-          "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing"
-      }
-    }
-
     outputs = outputs.collectEntries { name, value ->
       def par = config.allArguments.find { it.plainName == name && it.direction == "output" }
       assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid output argument"
@@ -213,6 +201,16 @@ Map _processOutputValues(Map outputs, Map config, String id, String key) {
   return outputs
 }
 
+void _checkAllRequiredOuputsPresent(Map outputs, Map config, String id, String key) {
+  if (!workflow.stubRun) {
+    config.allArguments.each { arg ->
+      if (arg.direction == "output" && arg.required) {
+        assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : 
+          "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing"
+      }
+    }
+  }
+}
 // helper file: 'src/main/resources/io/viash/runners/nextflow/channel/IDChecker.nf'
 class IDChecker {
   final def items = [] as Set
@@ -1666,6 +1664,162 @@ def joinStates(Closure apply_) {
   }
   return joinStatesWf
 }
+// helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishFiles.nf'
+def publishFiles(Map args) {
+  def key_ = args.get("key")
+
+  assert key_ != null : "publishFiles: key must be specified"
+  
+  workflow publishFilesWf {
+    take: input_ch
+    main:
+      input_ch
+        | map { tup ->
+          def id_ = tup[0]
+          def state_ = tup[1]
+
+          // the input files and the target output filenames
+          def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose()
+          def inputFiles_ = inputoutputFilenames_[0]
+          def outputFilenames_ = inputoutputFilenames_[1]
+
+          [id_, inputFiles_, outputFilenames_]
+        }
+        | publishFilesProc
+    emit: input_ch
+  }
+  return publishFilesWf
+}
+
+process publishFilesProc {
+  // todo: check publishpath?
+  publishDir path: "${getPublishDir()}/", mode: "copy"
+  tag "$id"
+  input:
+    tuple val(id), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles)
+  output:
+    tuple val(id), path{outputFiles}
+  script:
+  def copyCommands = [
+    inputFiles instanceof List ? inputFiles : [inputFiles],
+    outputFiles instanceof List ? outputFiles : [outputFiles]
+  ]
+    .transpose()
+    .collectMany{infile, outfile ->
+      if (infile.toString() != outfile.toString()) {
+        [
+          "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"",
+          "cp -r '${infile.toString()}' '${outfile.toString()}'"
+        ]
+      } else {
+        // no need to copy if infile is the same as outfile
+        []
+      }
+    }
+  """
+  echo "Copying output files to destination folder"
+  ${copyCommands.join("\n  ")}
+  """
+}
+
+
+// this assumes that the state contains no other values other than those specified in the config
+def publishFilesByConfig(Map args) {
+  def config = args.get("config")
+  assert config != null : "publishFilesByConfig: config must be specified"
+
+  def key_ = args.get("key", config.name)
+  assert key_ != null : "publishFilesByConfig: key must be specified"
+  
+  workflow publishFilesSimpleWf {
+    take: input_ch
+    main:
+      input_ch
+        | map { tup ->
+          def id_ = tup[0]
+          def state_ = tup[1] // e.g. [output: new File("myoutput.h5ad"), k: 10]
+          def origState_ = tup[2] // e.g. [output: '$id.$key.foo.h5ad']
+
+
+          // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where
+          //   - key is a String
+          //   - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path)
+          //   - inputPath is a List[Path]
+          //   - outputFilename is a List[String]
+          //   - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml)
+          def processedState =
+            config.allArguments
+              .findAll { it.direction == "output" }
+              .collectMany { par ->
+                def plainName_ = par.plainName
+                // if the state does not contain the key, it's an
+                // optional argument for which the component did 
+                // not generate any output OR multiple channels were emitted
+                // and the output was just not added to using the channel
+                // that is now being parsed
+                if (!state_.containsKey(plainName_)) {
+                  return []
+                }
+                def value = state_[plainName_]
+                // if the parameter is not a file, it should be stored
+                // in the state as-is, but is not something that needs 
+                // to be copied from the source path to the dest path
+                if (par.type != "file") {
+                  return [[inputPath: [], outputFilename: []]]
+                }
+                // if the orig state does not contain this filename,
+                // it's an optional argument for which the user specified
+                // that it should not be returned as a state
+                if (!origState_.containsKey(plainName_)) {
+                  return []
+                }
+                def filenameTemplate = origState_[plainName_]
+                // if the pararameter is multiple: true, fetch the template
+                if (par.multiple && filenameTemplate instanceof List) {
+                  filenameTemplate = filenameTemplate[0]
+                }
+                // instantiate the template
+                def filename = filenameTemplate
+                  .replaceAll('\\$id', id_)
+                  .replaceAll('\\$\\{id\\}', id_)
+                  .replaceAll('\\$key', key_)
+                  .replaceAll('\\$\\{key\\}', key_)
+                if (par.multiple) {
+                  // if the parameter is multiple: true, the filename
+                  // should contain a wildcard '*' that is replaced with
+                  // the index of the file
+                  assert filename.contains("*") : "Module '${key_}' id '${id_}': Multiple output files specified, but no wildcard '*' in the filename: ${filename}"
+                  def outputPerFile = value.withIndex().collect{ val, ix ->
+                    def filename_ix = filename.replace("*", ix.toString())
+                    def inputPath = val instanceof File ? val.toPath() : val
+                    [inputPath: inputPath, outputFilename: filename_ix]
+                  }
+                  def transposedOutputs = ["inputPath", "outputFilename"].collectEntries{ key -> 
+                    [key, outputPerFile.collect{dic -> dic[key]}]
+                  }
+                  return [[key: plainName_] + transposedOutputs]
+                } else {
+                  def value_ = java.nio.file.Paths.get(filename)
+                  def inputPath = value instanceof File ? value.toPath() : value
+                  return [[inputPath: [inputPath], outputFilename: [filename]]]
+                }
+              }
+          
+          def inputPaths = processedState.collectMany{it.inputPath}
+          def outputFilenames = processedState.collectMany{it.outputFilename}
+          
+
+          [id_, inputPaths, outputFilenames]
+        }
+        | publishFilesProc
+    emit: input_ch
+  }
+  return publishFilesSimpleWf
+}
+
+
+
+
 // helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishStates.nf'
 def collectFiles(obj) {
   if (obj instanceof java.io.File || obj instanceof Path)  {
@@ -1723,8 +1877,6 @@ def publishStates(Map args) {
 
           // the input files and the target output filenames
           def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose()
-          def inputFiles_ = inputoutputFilenames_[0]
-          def outputFilenames_ = inputoutputFilenames_[1]
 
           def yamlFilename = yamlTemplate_
             .replaceAll('\\$id', id_)
@@ -1737,7 +1889,7 @@ def publishStates(Map args) {
           // convert state to yaml blob
           def yamlBlob_ = toRelativeTaggedYamlBlob([id: id_] + state_, java.nio.file.Paths.get(yamlFilename))
 
-          [id_, yamlBlob_, yamlFilename, inputFiles_, outputFilenames_]
+          [id_, yamlBlob_, yamlFilename]
         }
         | publishStatesProc
     emit: input_ch
@@ -1749,33 +1901,17 @@ process publishStatesProc {
   publishDir path: "${getPublishDir()}/", mode: "copy"
   tag "$id"
   input:
-    tuple val(id), val(yamlBlob), val(yamlFile), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles)
+    tuple val(id), val(yamlBlob), val(yamlFile)
   output:
-    tuple val(id), path{[yamlFile] + outputFiles}
+    tuple val(id), path{[yamlFile]}
   script:
-  def copyCommands = [
-    inputFiles instanceof List ? inputFiles : [inputFiles],
-    outputFiles instanceof List ? outputFiles : [outputFiles]
-  ]
-    .transpose()
-    .collectMany{infile, outfile ->
-      if (infile.toString() != outfile.toString()) {
-        [
-          "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"",
-          "cp -r '${infile.toString()}' '${outfile.toString()}'"
-        ]
-      } else {
-        // no need to copy if infile is the same as outfile
-        []
-      }
-    }
   """
-mkdir -p "\$(dirname '${yamlFile}')"
+  mkdir -p "\$(dirname '${yamlFile}')"
-echo "Storing state as yaml"
+  echo "Storing state as yaml"
-echo '${yamlBlob}' > '${yamlFile}'
+  cat > '${yamlFile}' << HERE
-echo "Copying output files to destination folder"
+${yamlBlob}
-${copyCommands.join("\n  ")}
+HERE
-"""
+  """
 }
 
 
@@ -1806,13 +1942,10 @@ def publishStatesByConfig(Map args) {
             .replaceAll('\\$\\{key\\}', key_)
           def yamlDir = java.nio.file.Paths.get(yamlFilename).getParent()
 
-          // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where
+          // the processed state is a list of [key, value] tuples, where
           //   - key is a String
           //   - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path)
-          //   - inputPath is a List[Path]
-          //   - outputFilename is a List[String]
           //   - (key, value) are the tuples that will be saved to the state.yaml file
-          //   - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml)
           def processedState =
             config.allArguments
               .findAll { it.direction == "output" }
@@ -1829,7 +1962,7 @@ def publishStatesByConfig(Map args) {
                 // in the state as-is, but is not something that needs 
                 // to be copied from the source path to the dest path
                 if (par.type != "file") {
-                  return [[key: plainName_, value: value, inputPath: [], outputFilename: []]]
+                  return [[key: plainName_, value: value]]
                 }
                 // if the orig state does not contain this filename,
                 // it's an optional argument for which the user specified
@@ -1860,13 +1993,9 @@ def publishStatesByConfig(Map args) {
                     if (yamlDir != null) {
                       value_ = yamlDir.relativize(value_)
                     }
-                    def inputPath = val instanceof File ? val.toPath() : val
+                    return value_
-                    [value: value_, inputPath: inputPath, outputFilename: filename_ix]
                   }
-                  def transposedOutputs = ["value", "inputPath", "outputFilename"].collectEntries{ key -> 
+                  return [["key": plainName_, "value": outputPerFile]]
-                    [key, outputPerFile.collect{dic -> dic[key]}]
-                  }
-                  return [[key: plainName_] + transposedOutputs]
                 } else {
                   def value_ = java.nio.file.Paths.get(filename)
                   // if id contains a slash
@@ -1874,18 +2003,17 @@ def publishStatesByConfig(Map args) {
                     value_ = yamlDir.relativize(value_)
                   }
                   def inputPath = value instanceof File ? value.toPath() : value
-                  return [[key: plainName_, value: value_, inputPath: [inputPath], outputFilename: [filename]]]
+                  return [["key": plainName_, value: value_]]
                 }
               }
+              
           
           def updatedState_ = processedState.collectEntries{[it.key, it.value]}
-          def inputPaths = processedState.collectMany{it.inputPath}
-          def outputFilenames = processedState.collectMany{it.outputFilename}
           
           // convert state to yaml blob
           def yamlBlob_ = toTaggedYamlBlob([id: id_] + updatedState_)
 
-          [id_, yamlBlob_, yamlFilename, inputPaths, outputFilenames]
+          [id_, yamlBlob_, yamlFilename]
         }
         | publishStatesProc
     emit: input_ch
@@ -2559,7 +2687,8 @@ def _debug(workflowArgs, debugKey) {
 def workflowFactory(Map args, Map defaultWfArgs, Map meta) {
   def workflowArgs = processWorkflowArgs(args, defaultWfArgs, meta)
   def key_ = workflowArgs["key"]
-  
+  def multipleArgs = meta.config.allArguments.findAll{ it.multiple }.collect{it.plainName}
+
   workflow workflowInstance {
     take: input_
 
@@ -2716,12 +2845,36 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) {
       }
 
     // TODO: move some of the _meta.join_id wrangling to the safeJoin() function.
-    def chInitialOutput = chArgsWithDefaults
+    def chInitialOutputMulti = chArgsWithDefaults
       | _debug(workflowArgs, "processed")
       // run workflow
       | innerWorkflowFactory(workflowArgs)
-      // check output tuple
+    def chInitialOutputList = chInitialOutputMulti instanceof List ? chInitialOutputMulti : [chInitialOutputMulti]
-      | map { id_, output_ ->
+    assert chInitialOutputList.size() > 0: "should have emitted at least one output channel"
+    // Add a channel ID to the events, which designates the channel the event was emitted from as a running number
+    // This number is used to sort the events later when the events are gathered from across the channels.
+    def chInitialOutputListWithIndexedEvents = chInitialOutputList.withIndex().collect{channel, channelIndex ->
+      def newChannel = channel
+        | map {tuple ->
+          assert tuple instanceof List : 
+          "Error in module '${key_}': element in output channel should be a tuple [id, data, ...otherargs...]\n" +
+          "  Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" +
+          "  Expected class: List. Found: tuple.getClass() is ${tuple.getClass()}"
+        
+          def newEvent = [channelIndex] + tuple
+          return newEvent
+        }
+      return newChannel
+    }
+    // Put the events into 1 channel, cover case where there is only one channel is emitted
+    def chInitialOutput = chInitialOutputList.size() > 1 ? \
+      chInitialOutputListWithIndexedEvents[0].mix(*chInitialOutputListWithIndexedEvents.tail()) : \
+      chInitialOutputListWithIndexedEvents[0]
+    def chInitialOutputProcessed = chInitialOutput
+      | map { tuple  ->
+        def channelId = tuple[0]
+        def id_ = tuple[1]
+        def output_ = tuple[2]
 
         // see if output map contains metadata
         def meta_ =
@@ -2734,19 +2887,94 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) {
         output_ = output_.findAll{k, v -> k != "_meta"}
 
         // check value types
-        output_ = _processOutputValues(output_, meta.config, id_, key_)
+        output_ = _checkValidOutputArgument(output_, meta.config, id_, key_)
 
-        // simplify output if need be
+        [join_id, channelId, id_, output_]
-        if (workflowArgs.auto.simplifyOutput && output_.size() == 1) {
-          output_ = output_.values()[0]
-        }
-
-        [join_id, id_, output_]
       }
       // | view{"chInitialOutput: ${it.take(3)}"}
 
+    // join the output [prev_id, channel_id, new_id, output] with the previous state [prev_id, state, ...]
+    def chPublishWithPreviousState = safeJoin(chInitialOutputProcessed, chRunFiltered, key_)
+      // input tuple format: [join_id, channel_id, id, output, prev_state, ...]
+      // output tuple format: [join_id, channel_id, id, new_state, ...]
+      | map{ tup ->
+        def new_state = workflowArgs.toState(tup.drop(2).take(3))
+        tup.take(3) + [new_state] + tup.drop(5)
+      }
+    if (workflowArgs.auto.publish == "state") {
+      def chPublishFiles = chPublishWithPreviousState
+        // input tuple format: [join_id, channel_id, id, new_state, ...]
+        // output tuple format: [join_id, channel_id, id, new_state]
+        | map{ tup ->
+          tup.take(4)
+        }
+
+      safeJoin(chPublishFiles, chArgsWithDefaults, key_)
+        // input tuple format: [join_id, channel_id, id, new_state, orig_state, ...]
+        // output tuple format: [id, new_state, orig_state]
+        | map { tup ->
+          tup.drop(2).take(3)
+        }
+        | publishFilesByConfig(key: key_, config: meta.config)
+    }
+    // Join the state from the events that were emitted from different channels
+    def chJoined = chInitialOutputProcessed
+      | map {tuple ->
+        def join_id = tuple[0]
+        def channel_id = tuple[1]
+        def id = tuple[2]
+        def other = tuple.drop(3)
+        // Below, groupTuple is used to join the events. To make sure resuming a workflow
+        // keeps working, the output state must be deterministic. This means the state needs to be
+        // sorted with groupTuple's has a 'sort' argument. This argument can be set to 'hash',
+        // but hashing the state when it is large can be problematic in terms of performance.
+        // Therefore, a custom comparator function is provided. We add the channel ID to the 
+        // states so that we can use the channel ID to sort the items. 
+        def stateWithChannelID = [[channel_id] * other.size(), other].transpose()
+        // A comparator that is provided to groupTuple's 'sort' argument is applied
+        // to all elements of the event tuple (that is not the 'id'). The comparator
+        // closure that is used below expects the input to be List. So the join_id and
+        // channel_id must also be wrapped in a list. 
+        [[join_id], [channel_id], id] + stateWithChannelID
+      }
+      | groupTuple(by: 2, sort: {a, b -> a[0] <=> b[0]}, size: chInitialOutputList.size(), remainder: true)
+      | map {join_ids, _, id, statesWithChannelID ->
+        // Remove the channel IDs from the states
+        def states = statesWithChannelID.collect{it[1]}
+        def newJoinId = join_ids.flatten().unique{a, b -> a <=> b}
+        assert newJoinId.size() == 1: "Multiple events were emitted for '$id'."
+        def newJoinIdUnique = newJoinId[0]
+        
+        // Merge the states from the different channels
+        def newState = states.inject([:]){ old_state, state_to_add ->
+          return old_state + state_to_add.collectEntries{k, v -> 
+            if (!multipleArgs.contains(k)) {
+              // if the key is not a multiple argument, we expect only one value
+              if (old_state.containsKey(k)) {
+                assert old_state[k] == v : "ID $id: multiple entries for argument $k were emitted."
+              }
+              [k, v]
+            } else {
+              // if the key is a multiple argument, append the different values into one list
+              def prevValue = old_state.getOrDefault(k, [])
+              def prevValueAsList = prevValue instanceof List ? prevValue : [prevValue]
+              [k, prevValueAsList + v]
+            }
+          }
+        }
+
+        _checkAllRequiredOuputsPresent(newState, meta.config, id, key_)
+
+        // simplify output if need be
+        if (workflowArgs.auto.simplifyOutput && newState.size() == 1) {
+          newState = newState.values()[0]
+        }
+
+        return [newJoinIdUnique, id, newState]
+      }
+    
     // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...]
-    def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_)
+    def chNewState = safeJoin(chJoined, chRunFiltered, key_)
       // input tuple format: [join_id, id, output, prev_state, ...]
       // output tuple format: [join_id, id, new_state, ...]
       | map{ tup ->
@@ -2755,23 +2983,21 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) {
       }
 
     if (workflowArgs.auto.publish == "state") {
-      def chPublish = chNewState
+      def chPublishStates = chNewState
         // input tuple format: [join_id, id, new_state, ...]
         // output tuple format: [join_id, id, new_state]
         | map{ tup ->
           tup.take(3)
         }
 
-      safeJoin(chPublish, chArgsWithDefaults, key_)
+      safeJoin(chPublishStates, chArgsWithDefaults, key_)
         // input tuple format: [join_id, id, new_state, orig_state, ...]
         // output tuple format: [id, new_state, orig_state]
         | map { tup ->
           tup.drop(1).take(3)
-      }
+        }
         | publishStatesByConfig(key: key_, config: meta.config)
     }
-
-    // remove join_id and meta
     chReturn = chNewState
       | map { tup ->
         // input tuple format: [join_id, id, new_state, ...]
@@ -2806,7 +3032,7 @@ meta = [
   "config": processConfig(readJsonBlob('''{
   "name" : "untar",
   "namespace" : "io",
-  "version" : "v0.3.7",
+  "version" : "v0.3.9",
   "argument_groups" : [
     {
       "name" : "Input arguments",
@@ -2882,6 +3108,10 @@ meta = [
     }
   ],
   "status" : "enabled",
+  "scope" : {
+    "image" : "public",
+    "target" : "public"
+  },
   "requirements" : {
     "commands" : [
       "ps"
@@ -2974,7 +3204,7 @@ meta = [
       "id" : "docker",
       "image" : "debian:stable-slim",
       "target_registry" : "images.viash-hub.com",
-      "target_tag" : "v0.3.7",
+      "target_tag" : "v0.3.9",
       "namespace_separator" : "/",
       "setup" : [
         {
@@ -2996,14 +3226,14 @@ meta = [
     "runner" : "nextflow",
     "engine" : "docker|native",
     "output" : "target/nextflow/io/untar",
-    "viash_version" : "0.9.0",
+    "viash_version" : "0.9.4",
-    "git_commit" : "e177226c52ca1dd5a5cf9bb0bfc108812e2aa332",
+    "git_commit" : "820318c378d8119e2aa98768282e43c2aa017ba7",
     "git_remote" : "https://github.com/viash-hub/demultiplex",
-    "git_tag" : "v0.3.5-6-ge177226"
+    "git_tag" : "v0.3.8-5-g820318c"
   },
   "package_config" : {
     "name" : "demultiplex",
-    "version" : "v0.3.7",
+    "version" : "v0.3.9",
     "description" : "Demultiplexing pipeline\n",
     "info" : {
       "test_resources" : [
@@ -3013,14 +3243,14 @@ meta = [
         }
       ]
     },
-    "viash_version" : "0.9.0",
+    "viash_version" : "0.9.4",
     "source" : "src",
     "target" : "target",
     "config_mods" : [
-      ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n",
+      ".requirements.commands += ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n",
       ".engines += { type: \\"native\\" }",
       ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'",
-      ".engines[.type == 'docker'].target_tag := 'v0.3.7'"
+      ".engines[.type == 'docker'].target_tag := 'v0.3.9'"
     ],
     "keywords" : [
       "bioinformatics",
@@ -3446,7 +3676,7 @@ def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) {
   // create process from temp file
   def binding = new nextflow.script.ScriptBinding([:])
   def session = nextflow.Nextflow.getSession()
-  def parser = new nextflow.script.ScriptParser(session)
+  def parser = _getScriptLoader(session)
     .setModule(true)
     .setBinding(binding)
   def moduleScript = parser.runScript(tempFile)
@@ -3460,6 +3690,27 @@ def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) {
   return scriptMeta.getProcess(procKey)
 }
 
+// use Reflection to get a ScriptParser / ScriptLoader
+//   <25.02.0-edge: new nextflow.script.ScriptParser(session)
+//   >=25.02.0-edge: nextflow.script.ScriptLoaderFactory.create(session)
+def _getScriptLoader(nextflow.Session session) {
+  // try using the old method
+  try {
+    Class<?> scriptParserClass = Class.forName('nextflow.script.ScriptParser')
+    return scriptParserClass.getDeclaredConstructor(nextflow.Session).newInstance(session)
+  } catch (ClassNotFoundException e) {
+    // else try with the new method
+    try {
+      Class<?> scriptLoaderFactoryClass = Class.forName('nextflow.script.ScriptLoaderFactory')
+      def createMethod = scriptLoaderFactoryClass.getDeclaredMethod('create', nextflow.Session)
+      return createMethod.invoke(null, session) // null because create is static
+    } catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException | java.lang.reflect.InvocationTargetException e2) {
+      // Handle the case where neither class is found
+      throw new Exception("Neither nextflow.script.ScriptParser nor nextflow.script.ScriptLoaderFactory could be found. Is this a compatible Nextflow version?", e2)
+    }
+  }
+}
+
 // defaults
 meta["defaults"] = [
   // key to be used to trace the process and determine output names
@@ -3473,7 +3724,7 @@ meta["defaults"] = [
   "container" : {
     "registry" : "images.viash-hub.com",
     "image" : "vsh/demultiplex/io/untar",
-    "tag" : "v0.3.7"
+    "tag" : "v0.3.9"
   },
   "tag" : "$id"
 }'''),

target/nextflow/io/untar/nextflow.config

@@ -2,7 +2,7 @@ manifest {
   name = 'io/untar'
   mainScript = 'main.nf'
   nextflowVersion = '!>=20.12.1-edge'
-  version = 'v0.3.7'
+  version = 'v0.3.9'
   description = 'Unpack a .tar file. When the contents of the .tar file is just a single directory,\nput the contents of the directory into the output folder instead of that directory.\n'
 }
 

target/nextflow/io/untar/nextflow_schema.json

@@ -37,10 +37,10 @@
                 "output": {
                 "type":
                 "string",
-                "description": "Type: `file`, required, default: `$id.$key.output.output`. Directory to write the contents of the ",
+                "description": "Type: `file`, required, default: `$id.$key.output`. Directory to write the contents of the ",
-                "help_text": "Type: `file`, required, default: `$id.$key.output.output`. Directory to write the contents of the .tar file to."
+                "help_text": "Type: `file`, required, default: `$id.$key.output`. Directory to write the contents of the .tar file to."
             ,
-                "default":"$id.$key.output.output"
+                "default":"$id.$key.output"
             }
     
 

target/nextflow/runner/.config.vsh.yaml

@@ -1,5 +1,5 @@
 name: "runner"
-version: "v0.3.7"
+version: "v0.3.9"
 argument_groups:
 - name: "Input arguments"
   arguments:
@@ -103,6 +103,9 @@ resources:
 description: "Runner for demultiplexing of raw sequencing data"
 info: null
 status: "enabled"
+scope:
+  image: "public"
+  target: "public"
 requirements:
   commands:
   - "ps"
@@ -191,32 +194,32 @@ build_info:
   engine: "native|native"
   output: "target/nextflow/runner"
   executable: "target/nextflow/runner/main.nf"
-  viash_version: "0.9.0"
+  viash_version: "0.9.4"
-  git_commit: "e177226c52ca1dd5a5cf9bb0bfc108812e2aa332"
+  git_commit: "820318c378d8119e2aa98768282e43c2aa017ba7"
   git_remote: "https://github.com/viash-hub/demultiplex"
-  git_tag: "v0.3.5-6-ge177226"
+  git_tag: "v0.3.8-5-g820318c"
   dependencies:
   - "target/nextflow/demultiplex"
   - "target/nextflow/io/publish"
 package_config:
   name: "demultiplex"
-  version: "v0.3.7"
+  version: "v0.3.9"
   description: "Demultiplexing pipeline\n"
   info:
     test_resources:
     - path: "gs://viash-hub-test-data/demultiplex/v2/"
       dest: "testData"
-  viash_version: "0.9.0"
+  viash_version: "0.9.4"
   source: "src"
   target: "target"
   config_mods:
-  - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\
+  - ".requirements.commands += ['ps']\n.runners[.type == 'nextflow'].directives.tag\
     \ := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n\
     .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\
     )'\n"
   - ".engines += { type: \"native\" }"
   - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
-  - ".engines[.type == 'docker'].target_tag := 'v0.3.7'"
+  - ".engines[.type == 'docker'].target_tag := 'v0.3.9'"
   keywords:
   - "bioinformatics"
   - "sequence"

target/nextflow/runner/main.nf

@@ -1,6 +1,6 @@
-// runner v0.3.7
+// runner v0.3.9
 // 
-// This wrapper script is auto-generated by viash 0.9.0 and is thus a derivative
+// This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative
 // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data
 // Intuitive.
 // 
@@ -82,64 +82,56 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi
       foundClass = "List[${e.foundClass}]"
     }
   } else if (par.type == "string") {
-    // cast to string if need be
+    // cast to string if need be. only cast if the value is a GString
     if (value instanceof GString) {
-      value = value.toString()
+      value = value as String
     }
     expectedClass = value instanceof String ? null : "String"
   } else if (par.type == "integer") {
     // cast to integer if need be
-    if (value instanceof String) {
+    if (value !instanceof Integer) {
       try {
-        value = value.toInteger()
+        value = value as Integer
       } catch (NumberFormatException e) {
-        // do nothing
+        expectedClass = "Integer"
       }
     }
-    if (value instanceof java.math.BigInteger) {
-      value = value.intValue()
-    }
-    expectedClass = value instanceof Integer ? null : "Integer"
   } else if (par.type == "long") {
     // cast to long if need be
-    if (value instanceof String) {
+    if (value !instanceof Long) {
       try {
-        value = value.toLong()
+        value = value as Long
       } catch (NumberFormatException e) {
-        // do nothing
+        expectedClass = "Long"
       }
     }
-    if (value instanceof Integer) {
-      value = value.toLong()
-    }
-    expectedClass = value instanceof Long ? null : "Long"
   } else if (par.type == "double") {
     // cast to double if need be
-    if (value instanceof String) {
+    if (value !instanceof Double) {
       try {
-        value = value.toDouble()
+        value = value as Double
       } catch (NumberFormatException e) {
-        // do nothing
+        expectedClass = "Double"
       }
     }
-    if (value instanceof java.math.BigDecimal) {
+  } else if (par.type == "float") {
-      value = value.doubleValue()
+    // cast to float if need be
+    if (value !instanceof Float) {
+      try {
+        value = value as Float
+      } catch (NumberFormatException e) {
+        expectedClass = "Float"
+      }
     }
-    if (value instanceof Float) {
-      value = value.toDouble()
-    }
-    expectedClass = value instanceof Double ? null : "Double"
   } else if (par.type == "boolean" | par.type == "boolean_true" | par.type == "boolean_false") {
     // cast to boolean if need be
-    if (value instanceof String) {
+    if (value !instanceof Boolean) {
-      def valueLower = value.toLowerCase()
+      try {
-      if (valueLower == "true") {
+        value = value as Boolean
-        value = true
+      } catch (Exception e) {
-      } else if (valueLower == "false") {
+        expectedClass = "Boolean"
-        value = false
       }
     }
-    expectedClass = value instanceof Boolean ? null : "Boolean"
   } else if (par.type == "file" && (par.direction == "input" || stage == "output")) {
     // cast to path if need be
     if (value instanceof String) {
@@ -151,10 +143,13 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi
     expectedClass = value instanceof Path ? null : "Path"
   } else if (par.type == "file" && stage == "input" && par.direction == "output") {
     // cast to string if need be
-    if (value instanceof GString) {
+    if (value !instanceof String) {
-      value = value.toString()
+      try {
+        value = value as String
+      } catch (Exception e) {
+        expectedClass = "String"
+      }
     }
-    expectedClass = value instanceof String ? null : "String"
   } else {
     // didn't find a match for par.type
     expectedClass = par.type
@@ -173,7 +168,7 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi
 Map _processInputValues(Map inputs, Map config, String id, String key) {
   if (!workflow.stubRun) {
     config.allArguments.each { arg ->
-      if (arg.required) {
+      if (arg.required && arg.direction == "input") {
         assert inputs.containsKey(arg.plainName) && inputs.get(arg.plainName) != null : 
           "Error in module '${key}' id '${id}': required input argument '${arg.plainName}' is missing"
       }
@@ -192,15 +187,8 @@ Map _processInputValues(Map inputs, Map config, String id, String key) {
 }
 
 // helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_processOutputValues.nf'
-Map _processOutputValues(Map outputs, Map config, String id, String key) {
+Map _checkValidOutputArgument(Map outputs, Map config, String id, String key) {
   if (!workflow.stubRun) {
-    config.allArguments.each { arg ->
-      if (arg.direction == "output" && arg.required) {
-        assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : 
-          "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing"
-      }
-    }
-
     outputs = outputs.collectEntries { name, value ->
       def par = config.allArguments.find { it.plainName == name && it.direction == "output" }
       assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid output argument"
@@ -213,6 +201,16 @@ Map _processOutputValues(Map outputs, Map config, String id, String key) {
   return outputs
 }
 
+void _checkAllRequiredOuputsPresent(Map outputs, Map config, String id, String key) {
+  if (!workflow.stubRun) {
+    config.allArguments.each { arg ->
+      if (arg.direction == "output" && arg.required) {
+        assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : 
+          "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing"
+      }
+    }
+  }
+}
 // helper file: 'src/main/resources/io/viash/runners/nextflow/channel/IDChecker.nf'
 class IDChecker {
   final def items = [] as Set
@@ -1666,6 +1664,162 @@ def joinStates(Closure apply_) {
   }
   return joinStatesWf
 }
+// helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishFiles.nf'
+def publishFiles(Map args) {
+  def key_ = args.get("key")
+
+  assert key_ != null : "publishFiles: key must be specified"
+  
+  workflow publishFilesWf {
+    take: input_ch
+    main:
+      input_ch
+        | map { tup ->
+          def id_ = tup[0]
+          def state_ = tup[1]
+
+          // the input files and the target output filenames
+          def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose()
+          def inputFiles_ = inputoutputFilenames_[0]
+          def outputFilenames_ = inputoutputFilenames_[1]
+
+          [id_, inputFiles_, outputFilenames_]
+        }
+        | publishFilesProc
+    emit: input_ch
+  }
+  return publishFilesWf
+}
+
+process publishFilesProc {
+  // todo: check publishpath?
+  publishDir path: "${getPublishDir()}/", mode: "copy"
+  tag "$id"
+  input:
+    tuple val(id), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles)
+  output:
+    tuple val(id), path{outputFiles}
+  script:
+  def copyCommands = [
+    inputFiles instanceof List ? inputFiles : [inputFiles],
+    outputFiles instanceof List ? outputFiles : [outputFiles]
+  ]
+    .transpose()
+    .collectMany{infile, outfile ->
+      if (infile.toString() != outfile.toString()) {
+        [
+          "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"",
+          "cp -r '${infile.toString()}' '${outfile.toString()}'"
+        ]
+      } else {
+        // no need to copy if infile is the same as outfile
+        []
+      }
+    }
+  """
+  echo "Copying output files to destination folder"
+  ${copyCommands.join("\n  ")}
+  """
+}
+
+
+// this assumes that the state contains no other values other than those specified in the config
+def publishFilesByConfig(Map args) {
+  def config = args.get("config")
+  assert config != null : "publishFilesByConfig: config must be specified"
+
+  def key_ = args.get("key", config.name)
+  assert key_ != null : "publishFilesByConfig: key must be specified"
+  
+  workflow publishFilesSimpleWf {
+    take: input_ch
+    main:
+      input_ch
+        | map { tup ->
+          def id_ = tup[0]
+          def state_ = tup[1] // e.g. [output: new File("myoutput.h5ad"), k: 10]
+          def origState_ = tup[2] // e.g. [output: '$id.$key.foo.h5ad']
+
+
+          // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where
+          //   - key is a String
+          //   - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path)
+          //   - inputPath is a List[Path]
+          //   - outputFilename is a List[String]
+          //   - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml)
+          def processedState =
+            config.allArguments
+              .findAll { it.direction == "output" }
+              .collectMany { par ->
+                def plainName_ = par.plainName
+                // if the state does not contain the key, it's an
+                // optional argument for which the component did 
+                // not generate any output OR multiple channels were emitted
+                // and the output was just not added to using the channel
+                // that is now being parsed
+                if (!state_.containsKey(plainName_)) {
+                  return []
+                }
+                def value = state_[plainName_]
+                // if the parameter is not a file, it should be stored
+                // in the state as-is, but is not something that needs 
+                // to be copied from the source path to the dest path
+                if (par.type != "file") {
+                  return [[inputPath: [], outputFilename: []]]
+                }
+                // if the orig state does not contain this filename,
+                // it's an optional argument for which the user specified
+                // that it should not be returned as a state
+                if (!origState_.containsKey(plainName_)) {
+                  return []
+                }
+                def filenameTemplate = origState_[plainName_]
+                // if the pararameter is multiple: true, fetch the template
+                if (par.multiple && filenameTemplate instanceof List) {
+                  filenameTemplate = filenameTemplate[0]
+                }
+                // instantiate the template
+                def filename = filenameTemplate
+                  .replaceAll('\\$id', id_)
+                  .replaceAll('\\$\\{id\\}', id_)
+                  .replaceAll('\\$key', key_)
+                  .replaceAll('\\$\\{key\\}', key_)
+                if (par.multiple) {
+                  // if the parameter is multiple: true, the filename
+                  // should contain a wildcard '*' that is replaced with
+                  // the index of the file
+                  assert filename.contains("*") : "Module '${key_}' id '${id_}': Multiple output files specified, but no wildcard '*' in the filename: ${filename}"
+                  def outputPerFile = value.withIndex().collect{ val, ix ->
+                    def filename_ix = filename.replace("*", ix.toString())
+                    def inputPath = val instanceof File ? val.toPath() : val
+                    [inputPath: inputPath, outputFilename: filename_ix]
+                  }
+                  def transposedOutputs = ["inputPath", "outputFilename"].collectEntries{ key -> 
+                    [key, outputPerFile.collect{dic -> dic[key]}]
+                  }
+                  return [[key: plainName_] + transposedOutputs]
+                } else {
+                  def value_ = java.nio.file.Paths.get(filename)
+                  def inputPath = value instanceof File ? value.toPath() : value
+                  return [[inputPath: [inputPath], outputFilename: [filename]]]
+                }
+              }
+          
+          def inputPaths = processedState.collectMany{it.inputPath}
+          def outputFilenames = processedState.collectMany{it.outputFilename}
+          
+
+          [id_, inputPaths, outputFilenames]
+        }
+        | publishFilesProc
+    emit: input_ch
+  }
+  return publishFilesSimpleWf
+}
+
+
+
+
 // helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishStates.nf'
 def collectFiles(obj) {
   if (obj instanceof java.io.File || obj instanceof Path)  {
@@ -1723,8 +1877,6 @@ def publishStates(Map args) {
 
           // the input files and the target output filenames
           def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose()
-          def inputFiles_ = inputoutputFilenames_[0]
-          def outputFilenames_ = inputoutputFilenames_[1]
 
           def yamlFilename = yamlTemplate_
             .replaceAll('\\$id', id_)
@@ -1737,7 +1889,7 @@ def publishStates(Map args) {
           // convert state to yaml blob
           def yamlBlob_ = toRelativeTaggedYamlBlob([id: id_] + state_, java.nio.file.Paths.get(yamlFilename))
 
-          [id_, yamlBlob_, yamlFilename, inputFiles_, outputFilenames_]
+          [id_, yamlBlob_, yamlFilename]
         }
         | publishStatesProc
     emit: input_ch
@@ -1749,33 +1901,17 @@ process publishStatesProc {
   publishDir path: "${getPublishDir()}/", mode: "copy"
   tag "$id"
   input:
-    tuple val(id), val(yamlBlob), val(yamlFile), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles)
+    tuple val(id), val(yamlBlob), val(yamlFile)
   output:
-    tuple val(id), path{[yamlFile] + outputFiles}
+    tuple val(id), path{[yamlFile]}
   script:
-  def copyCommands = [
-    inputFiles instanceof List ? inputFiles : [inputFiles],
-    outputFiles instanceof List ? outputFiles : [outputFiles]
-  ]
-    .transpose()
-    .collectMany{infile, outfile ->
-      if (infile.toString() != outfile.toString()) {
-        [
-          "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"",
-          "cp -r '${infile.toString()}' '${outfile.toString()}'"
-        ]
-      } else {
-        // no need to copy if infile is the same as outfile
-        []
-      }
-    }
   """
-mkdir -p "\$(dirname '${yamlFile}')"
+  mkdir -p "\$(dirname '${yamlFile}')"
-echo "Storing state as yaml"
+  echo "Storing state as yaml"
-echo '${yamlBlob}' > '${yamlFile}'
+  cat > '${yamlFile}' << HERE
-echo "Copying output files to destination folder"
+${yamlBlob}
-${copyCommands.join("\n  ")}
+HERE
-"""
+  """
 }
 
 
@@ -1806,13 +1942,10 @@ def publishStatesByConfig(Map args) {
             .replaceAll('\\$\\{key\\}', key_)
           def yamlDir = java.nio.file.Paths.get(yamlFilename).getParent()
 
-          // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where
+          // the processed state is a list of [key, value] tuples, where
           //   - key is a String
           //   - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path)
-          //   - inputPath is a List[Path]
-          //   - outputFilename is a List[String]
           //   - (key, value) are the tuples that will be saved to the state.yaml file
-          //   - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml)
           def processedState =
             config.allArguments
               .findAll { it.direction == "output" }
@@ -1829,7 +1962,7 @@ def publishStatesByConfig(Map args) {
                 // in the state as-is, but is not something that needs 
                 // to be copied from the source path to the dest path
                 if (par.type != "file") {
-                  return [[key: plainName_, value: value, inputPath: [], outputFilename: []]]
+                  return [[key: plainName_, value: value]]
                 }
                 // if the orig state does not contain this filename,
                 // it's an optional argument for which the user specified
@@ -1860,13 +1993,9 @@ def publishStatesByConfig(Map args) {
                     if (yamlDir != null) {
                       value_ = yamlDir.relativize(value_)
                     }
-                    def inputPath = val instanceof File ? val.toPath() : val
+                    return value_
-                    [value: value_, inputPath: inputPath, outputFilename: filename_ix]
                   }
-                  def transposedOutputs = ["value", "inputPath", "outputFilename"].collectEntries{ key -> 
+                  return [["key": plainName_, "value": outputPerFile]]
-                    [key, outputPerFile.collect{dic -> dic[key]}]
-                  }
-                  return [[key: plainName_] + transposedOutputs]
                 } else {
                   def value_ = java.nio.file.Paths.get(filename)
                   // if id contains a slash
@@ -1874,18 +2003,17 @@ def publishStatesByConfig(Map args) {
                     value_ = yamlDir.relativize(value_)
                   }
                   def inputPath = value instanceof File ? value.toPath() : value
-                  return [[key: plainName_, value: value_, inputPath: [inputPath], outputFilename: [filename]]]
+                  return [["key": plainName_, value: value_]]
                 }
               }
+              
           
           def updatedState_ = processedState.collectEntries{[it.key, it.value]}
-          def inputPaths = processedState.collectMany{it.inputPath}
-          def outputFilenames = processedState.collectMany{it.outputFilename}
           
           // convert state to yaml blob
           def yamlBlob_ = toTaggedYamlBlob([id: id_] + updatedState_)
 
-          [id_, yamlBlob_, yamlFilename, inputPaths, outputFilenames]
+          [id_, yamlBlob_, yamlFilename]
         }
         | publishStatesProc
     emit: input_ch
@@ -2559,7 +2687,8 @@ def _debug(workflowArgs, debugKey) {
 def workflowFactory(Map args, Map defaultWfArgs, Map meta) {
   def workflowArgs = processWorkflowArgs(args, defaultWfArgs, meta)
   def key_ = workflowArgs["key"]
-  
+  def multipleArgs = meta.config.allArguments.findAll{ it.multiple }.collect{it.plainName}
+
   workflow workflowInstance {
     take: input_
 
@@ -2716,12 +2845,36 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) {
       }
 
     // TODO: move some of the _meta.join_id wrangling to the safeJoin() function.
-    def chInitialOutput = chArgsWithDefaults
+    def chInitialOutputMulti = chArgsWithDefaults
       | _debug(workflowArgs, "processed")
       // run workflow
       | innerWorkflowFactory(workflowArgs)
-      // check output tuple
+    def chInitialOutputList = chInitialOutputMulti instanceof List ? chInitialOutputMulti : [chInitialOutputMulti]
-      | map { id_, output_ ->
+    assert chInitialOutputList.size() > 0: "should have emitted at least one output channel"
+    // Add a channel ID to the events, which designates the channel the event was emitted from as a running number
+    // This number is used to sort the events later when the events are gathered from across the channels.
+    def chInitialOutputListWithIndexedEvents = chInitialOutputList.withIndex().collect{channel, channelIndex ->
+      def newChannel = channel
+        | map {tuple ->
+          assert tuple instanceof List : 
+          "Error in module '${key_}': element in output channel should be a tuple [id, data, ...otherargs...]\n" +
+          "  Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" +
+          "  Expected class: List. Found: tuple.getClass() is ${tuple.getClass()}"
+        
+          def newEvent = [channelIndex] + tuple
+          return newEvent
+        }
+      return newChannel
+    }
+    // Put the events into 1 channel, cover case where there is only one channel is emitted
+    def chInitialOutput = chInitialOutputList.size() > 1 ? \
+      chInitialOutputListWithIndexedEvents[0].mix(*chInitialOutputListWithIndexedEvents.tail()) : \
+      chInitialOutputListWithIndexedEvents[0]
+    def chInitialOutputProcessed = chInitialOutput
+      | map { tuple  ->
+        def channelId = tuple[0]
+        def id_ = tuple[1]
+        def output_ = tuple[2]
 
         // see if output map contains metadata
         def meta_ =
@@ -2734,19 +2887,94 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) {
         output_ = output_.findAll{k, v -> k != "_meta"}
 
         // check value types
-        output_ = _processOutputValues(output_, meta.config, id_, key_)
+        output_ = _checkValidOutputArgument(output_, meta.config, id_, key_)
 
-        // simplify output if need be
+        [join_id, channelId, id_, output_]
-        if (workflowArgs.auto.simplifyOutput && output_.size() == 1) {
-          output_ = output_.values()[0]
-        }
-
-        [join_id, id_, output_]
       }
       // | view{"chInitialOutput: ${it.take(3)}"}
 
+    // join the output [prev_id, channel_id, new_id, output] with the previous state [prev_id, state, ...]
+    def chPublishWithPreviousState = safeJoin(chInitialOutputProcessed, chRunFiltered, key_)
+      // input tuple format: [join_id, channel_id, id, output, prev_state, ...]
+      // output tuple format: [join_id, channel_id, id, new_state, ...]
+      | map{ tup ->
+        def new_state = workflowArgs.toState(tup.drop(2).take(3))
+        tup.take(3) + [new_state] + tup.drop(5)
+      }
+    if (workflowArgs.auto.publish == "state") {
+      def chPublishFiles = chPublishWithPreviousState
+        // input tuple format: [join_id, channel_id, id, new_state, ...]
+        // output tuple format: [join_id, channel_id, id, new_state]
+        | map{ tup ->
+          tup.take(4)
+        }
+
+      safeJoin(chPublishFiles, chArgsWithDefaults, key_)
+        // input tuple format: [join_id, channel_id, id, new_state, orig_state, ...]
+        // output tuple format: [id, new_state, orig_state]
+        | map { tup ->
+          tup.drop(2).take(3)
+        }
+        | publishFilesByConfig(key: key_, config: meta.config)
+    }
+    // Join the state from the events that were emitted from different channels
+    def chJoined = chInitialOutputProcessed
+      | map {tuple ->
+        def join_id = tuple[0]
+        def channel_id = tuple[1]
+        def id = tuple[2]
+        def other = tuple.drop(3)
+        // Below, groupTuple is used to join the events. To make sure resuming a workflow
+        // keeps working, the output state must be deterministic. This means the state needs to be
+        // sorted with groupTuple's has a 'sort' argument. This argument can be set to 'hash',
+        // but hashing the state when it is large can be problematic in terms of performance.
+        // Therefore, a custom comparator function is provided. We add the channel ID to the 
+        // states so that we can use the channel ID to sort the items. 
+        def stateWithChannelID = [[channel_id] * other.size(), other].transpose()
+        // A comparator that is provided to groupTuple's 'sort' argument is applied
+        // to all elements of the event tuple (that is not the 'id'). The comparator
+        // closure that is used below expects the input to be List. So the join_id and
+        // channel_id must also be wrapped in a list. 
+        [[join_id], [channel_id], id] + stateWithChannelID
+      }
+      | groupTuple(by: 2, sort: {a, b -> a[0] <=> b[0]}, size: chInitialOutputList.size(), remainder: true)
+      | map {join_ids, _, id, statesWithChannelID ->
+        // Remove the channel IDs from the states
+        def states = statesWithChannelID.collect{it[1]}
+        def newJoinId = join_ids.flatten().unique{a, b -> a <=> b}
+        assert newJoinId.size() == 1: "Multiple events were emitted for '$id'."
+        def newJoinIdUnique = newJoinId[0]
+        
+        // Merge the states from the different channels
+        def newState = states.inject([:]){ old_state, state_to_add ->
+          return old_state + state_to_add.collectEntries{k, v -> 
+            if (!multipleArgs.contains(k)) {
+              // if the key is not a multiple argument, we expect only one value
+              if (old_state.containsKey(k)) {
+                assert old_state[k] == v : "ID $id: multiple entries for argument $k were emitted."
+              }
+              [k, v]
+            } else {
+              // if the key is a multiple argument, append the different values into one list
+              def prevValue = old_state.getOrDefault(k, [])
+              def prevValueAsList = prevValue instanceof List ? prevValue : [prevValue]
+              [k, prevValueAsList + v]
+            }
+          }
+        }
+
+        _checkAllRequiredOuputsPresent(newState, meta.config, id, key_)
+
+        // simplify output if need be
+        if (workflowArgs.auto.simplifyOutput && newState.size() == 1) {
+          newState = newState.values()[0]
+        }
+
+        return [newJoinIdUnique, id, newState]
+      }
+    
     // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...]
-    def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_)
+    def chNewState = safeJoin(chJoined, chRunFiltered, key_)
       // input tuple format: [join_id, id, output, prev_state, ...]
       // output tuple format: [join_id, id, new_state, ...]
       | map{ tup ->
@@ -2755,23 +2983,21 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) {
       }
 
     if (workflowArgs.auto.publish == "state") {
-      def chPublish = chNewState
+      def chPublishStates = chNewState
         // input tuple format: [join_id, id, new_state, ...]
         // output tuple format: [join_id, id, new_state]
         | map{ tup ->
           tup.take(3)
         }
 
-      safeJoin(chPublish, chArgsWithDefaults, key_)
+      safeJoin(chPublishStates, chArgsWithDefaults, key_)
         // input tuple format: [join_id, id, new_state, orig_state, ...]
         // output tuple format: [id, new_state, orig_state]
         | map { tup ->
           tup.drop(1).take(3)
-      }
+        }
         | publishStatesByConfig(key: key_, config: meta.config)
     }
-
-    // remove join_id and meta
     chReturn = chNewState
       | map { tup ->
         // input tuple format: [join_id, id, new_state, ...]
@@ -2805,7 +3031,7 @@ meta = [
   "resources_dir": moduleDir.toRealPath().normalize(),
   "config": processConfig(readJsonBlob('''{
   "name" : "runner",
-  "version" : "v0.3.7",
+  "version" : "v0.3.9",
   "argument_groups" : [
     {
       "name" : "Input arguments",
@@ -2929,6 +3155,10 @@ meta = [
   ],
   "description" : "Runner for demultiplexing of raw sequencing data",
   "status" : "enabled",
+  "scope" : {
+    "image" : "public",
+    "target" : "public"
+  },
   "requirements" : {
     "commands" : [
       "ps"
@@ -3039,14 +3269,14 @@ meta = [
     "runner" : "nextflow",
     "engine" : "native|native",
     "output" : "target/nextflow/runner",
-    "viash_version" : "0.9.0",
+    "viash_version" : "0.9.4",
-    "git_commit" : "e177226c52ca1dd5a5cf9bb0bfc108812e2aa332",
+    "git_commit" : "820318c378d8119e2aa98768282e43c2aa017ba7",
     "git_remote" : "https://github.com/viash-hub/demultiplex",
-    "git_tag" : "v0.3.5-6-ge177226"
+    "git_tag" : "v0.3.8-5-g820318c"
   },
   "package_config" : {
     "name" : "demultiplex",
-    "version" : "v0.3.7",
+    "version" : "v0.3.9",
     "description" : "Demultiplexing pipeline\n",
     "info" : {
       "test_resources" : [
@@ -3056,14 +3286,14 @@ meta = [
         }
       ]
     },
-    "viash_version" : "0.9.0",
+    "viash_version" : "0.9.4",
     "source" : "src",
     "target" : "target",
     "config_mods" : [
-      ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n",
+      ".requirements.commands += ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n",
       ".engines += { type: \\"native\\" }",
       ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'",
-      ".engines[.type == 'docker'].target_tag := 'v0.3.7'"
+      ".engines[.type == 'docker'].target_tag := 'v0.3.9'"
     ],
     "keywords" : [
       "bioinformatics",

target/nextflow/runner/nextflow.config

@@ -2,7 +2,7 @@ manifest {
   name = 'runner'
   mainScript = 'main.nf'
   nextflowVersion = '!>=20.12.1-edge'
-  version = 'v0.3.7'
+  version = 'v0.3.9'
   description = 'Runner for demultiplexing of raw sequencing data'
 }
 

target/nextflow/runner/nextflow_schema.json

@@ -80,10 +80,10 @@
                 "fastq_output": {
                 "type":
                 "string",
-                "description": "Type: `file`, default: `$id.$key.fastq_output.fastq_output`. ",
+                "description": "Type: `file`, default: `fastq`. ",
-                "help_text": "Type: `file`, default: `$id.$key.fastq_output.fastq_output`. "
+                "help_text": "Type: `file`, default: `fastq`. "
             ,
-                "default":"$id.$key.fastq_output.fastq_output"
+                "default":"fastq"
             }
     
 
@@ -91,10 +91,10 @@
                 "falco_output": {
                 "type":
                 "string",
-                "description": "Type: `file`, default: `$id.$key.falco_output.falco_output`. ",
+                "description": "Type: `file`, default: `qc/fastqc`. ",
-                "help_text": "Type: `file`, default: `$id.$key.falco_output.falco_output`. "
+                "help_text": "Type: `file`, default: `qc/fastqc`. "
             ,
-                "default":"$id.$key.falco_output.falco_output"
+                "default":"qc/fastqc"
             }
     
 
@@ -102,10 +102,10 @@
                 "multiqc_output": {
                 "type":
                 "string",
-                "description": "Type: `file`, default: `$id.$key.multiqc_output.html`. ",
+                "description": "Type: `file`, default: `qc/multiqc_report.html`. ",
-                "help_text": "Type: `file`, default: `$id.$key.multiqc_output.html`. "
+                "help_text": "Type: `file`, default: `qc/multiqc_report.html`. "
             ,
-                "default":"$id.$key.multiqc_output.html"
+                "default":"qc/multiqc_report.html"
             }