From 43010e8c5798d0d61ff9859ae39c5141b6f2f317 Mon Sep 17 00:00:00 2001 From: CI Date: Wed, 18 Dec 2024 15:15:49 +0000 Subject: [PATCH] Build branch main with version main (8d3c288) Build pipeline: viash-hub.demultiplex.main-mgtgd Source commit: https://github.com/viash-hub/demultiplex/commit/8d3c2888d6df6688763d303dba03ce53d339fb0f Source message: Improved output logic (#30) * Improved output logic * Strip suffix only + update description * Apply suggestions from code review Co-authored-by: Dries Schaumont <5946712+DriesSchaumont@users.noreply.github.com> --------- Co-authored-by: Dries Schaumont <5946712+DriesSchaumont@users.noreply.github.com> --- CHANGELOG.md | 51 +++++++++++++++++++ src/runner/config.vsh.yaml | 16 +++--- src/runner/main.nf | 22 +++++--- .../interop_summary_to_csv/.config.vsh.yaml | 6 +-- .../interop_summary_to_csv | 4 +- target/executable/io/publish/.config.vsh.yaml | 6 +-- target/executable/io/publish/publish | 4 +- target/executable/io/untar/.config.vsh.yaml | 6 +-- target/executable/io/untar/untar | 4 +- .../dataflow/combine_samples/.config.vsh.yaml | 6 +-- .../nextflow/dataflow/combine_samples/main.nf | 6 +-- .../.config.vsh.yaml | 6 +-- .../gather_fastqs_and_validate/main.nf | 6 +-- target/nextflow/demultiplex/.config.vsh.yaml | 6 +-- target/nextflow/demultiplex/main.nf | 6 +-- .../interop_summary_to_csv/.config.vsh.yaml | 6 +-- .../io/interop_summary_to_csv/main.nf | 6 +-- target/nextflow/io/publish/.config.vsh.yaml | 6 +-- target/nextflow/io/publish/main.nf | 6 +-- target/nextflow/io/untar/.config.vsh.yaml | 6 +-- target/nextflow/io/untar/main.nf | 6 +-- target/nextflow/runner/.config.vsh.yaml | 21 ++++---- target/nextflow/runner/main.nf | 40 ++++++++------- target/nextflow/runner/nextflow_schema.json | 21 ++------ 24 files changed, 161 insertions(+), 112 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3bb38b9..5d3f40b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,54 @@ +# demultiplex v0.3.3 + +## Breaking change + +- The `runner` defines the output differently now: + + - The last part of the `--input` path is expected to be the run ID and this run ID is used to create the output directory. + - If the input is `file.tar.gz` instead of a directory, the `file` part is used as the run ID. + +- The output structure is then as follows: + + ``` + $publish_dir//_demultiplex_/ + ``` + + For instance: + + ``` + $publish_dir + └── 200624_A00834_0183_BHMTFYDRXX + └── 20241217_051404_demultiplex_v1.2 + ├── fastq + │   ├── Sample1_S1_L001_R1_001.fastq.gz + │   ├── Sample23_S3_L001_R1_001.fastq.gz + │   ├── SampleA_S2_L001_R1_001.fastq.gz + │   ├── Undetermined_S0_L001_R1_001.fastq.gz + │   └── sampletest_S4_L001_R1_001.fastq.gz + └── qc + ├── fastqc + │   ├── Sample1_S1_L001_R1_001.fastq.gz_fastqc_data.txt + │   ├── Sample1_S1_L001_R1_001.fastq.gz_fastqc_report.html + │   ├── Sample1_S1_L001_R1_001.fastq.gz_summary.txt + │   ├── Sample23_S3_L001_R1_001.fastq.gz_fastqc_data.txt + │   ├── Sample23_S3_L001_R1_001.fastq.gz_fastqc_report.html + │   ├── Sample23_S3_L001_R1_001.fastq.gz_summary.txt + │   ├── SampleA_S2_L001_R1_001.fastq.gz_fastqc_data.txt + │   ├── SampleA_S2_L001_R1_001.fastq.gz_fastqc_report.html + │   ├── SampleA_S2_L001_R1_001.fastq.gz_summary.txt + │   ├── Undetermined_S0_L001_R1_001.fastq.gz_fastqc_data.txt + │   ├── Undetermined_S0_L001_R1_001.fastq.gz_fastqc_report.html + │   ├── Undetermined_S0_L001_R1_001.fastq.gz_summary.txt + │   ├── sampletest_S4_L001_R1_001.fastq.gz_fastqc_data.txt + │   ├── sampletest_S4_L001_R1_001.fastq.gz_fastqc_report.html + │   └── sampletest_S4_L001_R1_001.fastq.gz_summary.txt + └── multiqc_report.html + + ``` + +- This logic can be avoided by providing the flag `--plain_output`. + + # demultiplex v0.3.2 # Bug fixes diff --git a/src/runner/config.vsh.yaml b/src/runner/config.vsh.yaml index 50e4a1c..020be85 100644 --- a/src/runner/config.vsh.yaml +++ b/src/runner/config.vsh.yaml @@ -4,7 +4,10 @@ argument_groups: - name: Input arguments arguments: - name: --input - description: Base directory of the form `s3://Sequencing///` + description: | + Base directory of the canonical form `s3://///`. + A tarball (tar.gz, .tgz, .tar) containing run information can be provided in which + case the RunID is set to the name of the tarball without the extension. type: file required: true - name: --run_information @@ -27,15 +30,10 @@ argument_groups: required. - name: Annotation flags arguments: - - name: --add_date_time + - name: --plain_output description: | - Add date and time to the output directory name. This is useful - when running the same pipeline multiple times on the same input - directory. - type: boolean_true - - name: --add_workflow_id - description: | - Add a workflow identifier to the output directory name. + Flag to indicate that the output should be stored directly under $publish_dir rather than + under a subdirectory structure runID/_demultiplex_/. type: boolean_true - name: Output arguments arguments: diff --git a/src/runner/main.nf b/src/runner/main.nf index 5a92905..ef78f71 100644 --- a/src/runner/main.nf +++ b/src/runner/main.nf @@ -9,6 +9,15 @@ workflow run_wf { main: output_ch = input_ch + // Extract the ID from the input. + // If the input is a tarball, strip the suffix. + | map{ id, state -> + def id_with_suffix = state.input.getFileName().toString() + [ + id, + state + [ run_id: id_with_suffix - ~/\.(tar.gz|tgz|tar)$/ ] + ] + } | demultiplex.run( fromState: [ "input": "input", @@ -24,14 +33,15 @@ workflow run_wf { ) | publish.run( fromState: { id, state -> - def id1 = (params.add_date_time) ? "${id}_${date}" : id - def id2 = (params.add_workflow_id) ? "${id1}_demultiplex_${version}" : id1 + println(state.plain_output) + def id1 = (state.plain_output) ? id : "${state.run_id}/${date}" + def id2 = (state.plain_output) ? id : "${id1}_demultiplex_${version}" - def fastq_output_1 = (id == "run") ? state.fastq_output : "${id2}/" + state.fastq_output - def falco_output_1 = (id == "run") ? state.falco_output : "${id2}/" + state.falco_output - def multiqc_output_1 = (id == "run") ? state.multiqc_output : "${id2}/" + state.multiqc_output + def fastq_output_1 = (id2 == "run") ? state.fastq_output : "${id2}/" + state.fastq_output + def falco_output_1 = (id2 == "run") ? state.falco_output : "${id2}/" + state.falco_output + def multiqc_output_1 = (id2 == "run") ? state.multiqc_output : "${id2}/" + state.multiqc_output - if (id == "run") { + if (id2 == "run") { println("Publising to ${params.publish_dir}") } else { println("Publising to ${params.publish_dir}/${id2}") diff --git a/target/executable/io/interop_summary_to_csv/.config.vsh.yaml b/target/executable/io/interop_summary_to_csv/.config.vsh.yaml index 3cf0ef8..bdbd0c9 100644 --- a/target/executable/io/interop_summary_to_csv/.config.vsh.yaml +++ b/target/executable/io/interop_summary_to_csv/.config.vsh.yaml @@ -141,9 +141,9 @@ build_info: output: "target/executable/io/interop_summary_to_csv" executable: "target/executable/io/interop_summary_to_csv/interop_summary_to_csv" viash_version: "0.9.0" - git_commit: "45accaa50e52c6b29c78259995293789f345c80d" - git_remote: "https://x-access-token:ghs_Yd3of90BXneMupSK4qIbEdGLmCGJet1c95Fo@github.com/viash-hub/demultiplex" - git_tag: "v0.1.1-11-g45accaa" + git_commit: "8d3c2888d6df6688763d303dba03ce53d339fb0f" + git_remote: "https://x-access-token:ghs_Iv5SuazXszUy4WWFyKOtFABImrj2oY0GsE3i@github.com/viash-hub/demultiplex" + git_tag: "v0.1.1-12-g8d3c288" package_config: name: "demultiplex" version: "main" diff --git a/target/executable/io/interop_summary_to_csv/interop_summary_to_csv b/target/executable/io/interop_summary_to_csv/interop_summary_to_csv index 593ec31..7190766 100755 --- a/target/executable/io/interop_summary_to_csv/interop_summary_to_csv +++ b/target/executable/io/interop_summary_to_csv/interop_summary_to_csv @@ -470,9 +470,9 @@ tar -C /tmp/ --no-same-owner --no-same-permissions -xvf /tmp/interop.tar.gz && \ mv /tmp/interop-1.3.1-Linux-GNU/bin/index-summary /tmp/interop-1.3.1-Linux-GNU/bin/summary /usr/local/bin/ LABEL org.opencontainers.image.description="Companion container for running component io interop_summary_to_csv" -LABEL org.opencontainers.image.created="2024-12-11T17:49:17Z" +LABEL org.opencontainers.image.created="2024-12-18T15:00:53Z" LABEL org.opencontainers.image.source="https://github.com/viash-hub/demultiplex" -LABEL org.opencontainers.image.revision="45accaa50e52c6b29c78259995293789f345c80d" +LABEL org.opencontainers.image.revision="8d3c2888d6df6688763d303dba03ce53d339fb0f" LABEL org.opencontainers.image.version="main" VIASHDOCKER diff --git a/target/executable/io/publish/.config.vsh.yaml b/target/executable/io/publish/.config.vsh.yaml index f6bd2bb..a941315 100644 --- a/target/executable/io/publish/.config.vsh.yaml +++ b/target/executable/io/publish/.config.vsh.yaml @@ -170,9 +170,9 @@ build_info: output: "target/executable/io/publish" executable: "target/executable/io/publish/publish" viash_version: "0.9.0" - git_commit: "45accaa50e52c6b29c78259995293789f345c80d" - git_remote: "https://x-access-token:ghs_Yd3of90BXneMupSK4qIbEdGLmCGJet1c95Fo@github.com/viash-hub/demultiplex" - git_tag: "v0.1.1-11-g45accaa" + git_commit: "8d3c2888d6df6688763d303dba03ce53d339fb0f" + git_remote: "https://x-access-token:ghs_Iv5SuazXszUy4WWFyKOtFABImrj2oY0GsE3i@github.com/viash-hub/demultiplex" + git_tag: "v0.1.1-12-g8d3c288" package_config: name: "demultiplex" version: "main" diff --git a/target/executable/io/publish/publish b/target/executable/io/publish/publish index d7c1e80..e2b3070 100755 --- a/target/executable/io/publish/publish +++ b/target/executable/io/publish/publish @@ -482,9 +482,9 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* LABEL org.opencontainers.image.description="Companion container for running component io publish" -LABEL org.opencontainers.image.created="2024-12-11T17:49:17Z" +LABEL org.opencontainers.image.created="2024-12-18T15:00:54Z" LABEL org.opencontainers.image.source="https://github.com/viash-hub/demultiplex" -LABEL org.opencontainers.image.revision="45accaa50e52c6b29c78259995293789f345c80d" +LABEL org.opencontainers.image.revision="8d3c2888d6df6688763d303dba03ce53d339fb0f" LABEL org.opencontainers.image.version="main" VIASHDOCKER diff --git a/target/executable/io/untar/.config.vsh.yaml b/target/executable/io/untar/.config.vsh.yaml index 2450990..a0857e0 100644 --- a/target/executable/io/untar/.config.vsh.yaml +++ b/target/executable/io/untar/.config.vsh.yaml @@ -148,9 +148,9 @@ build_info: output: "target/executable/io/untar" executable: "target/executable/io/untar/untar" viash_version: "0.9.0" - git_commit: "45accaa50e52c6b29c78259995293789f345c80d" - git_remote: "https://x-access-token:ghs_Yd3of90BXneMupSK4qIbEdGLmCGJet1c95Fo@github.com/viash-hub/demultiplex" - git_tag: "v0.1.1-11-g45accaa" + git_commit: "8d3c2888d6df6688763d303dba03ce53d339fb0f" + git_remote: "https://x-access-token:ghs_Iv5SuazXszUy4WWFyKOtFABImrj2oY0GsE3i@github.com/viash-hub/demultiplex" + git_tag: "v0.1.1-12-g8d3c288" package_config: name: "demultiplex" version: "main" diff --git a/target/executable/io/untar/untar b/target/executable/io/untar/untar index 7ce066a..56a99cd 100755 --- a/target/executable/io/untar/untar +++ b/target/executable/io/untar/untar @@ -476,9 +476,9 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* LABEL org.opencontainers.image.description="Companion container for running component io untar" -LABEL org.opencontainers.image.created="2024-12-11T17:49:18Z" +LABEL org.opencontainers.image.created="2024-12-18T15:00:54Z" LABEL org.opencontainers.image.source="https://github.com/viash-hub/demultiplex" -LABEL org.opencontainers.image.revision="45accaa50e52c6b29c78259995293789f345c80d" +LABEL org.opencontainers.image.revision="8d3c2888d6df6688763d303dba03ce53d339fb0f" LABEL org.opencontainers.image.version="main" VIASHDOCKER diff --git a/target/nextflow/dataflow/combine_samples/.config.vsh.yaml b/target/nextflow/dataflow/combine_samples/.config.vsh.yaml index eb0c36e..08404f6 100644 --- a/target/nextflow/dataflow/combine_samples/.config.vsh.yaml +++ b/target/nextflow/dataflow/combine_samples/.config.vsh.yaml @@ -139,9 +139,9 @@ build_info: output: "target/nextflow/dataflow/combine_samples" executable: "target/nextflow/dataflow/combine_samples/main.nf" viash_version: "0.9.0" - git_commit: "45accaa50e52c6b29c78259995293789f345c80d" - git_remote: "https://x-access-token:ghs_Yd3of90BXneMupSK4qIbEdGLmCGJet1c95Fo@github.com/viash-hub/demultiplex" - git_tag: "v0.1.1-11-g45accaa" + git_commit: "8d3c2888d6df6688763d303dba03ce53d339fb0f" + git_remote: "https://x-access-token:ghs_Iv5SuazXszUy4WWFyKOtFABImrj2oY0GsE3i@github.com/viash-hub/demultiplex" + git_tag: "v0.1.1-12-g8d3c288" package_config: name: "demultiplex" version: "main" diff --git a/target/nextflow/dataflow/combine_samples/main.nf b/target/nextflow/dataflow/combine_samples/main.nf index ce9a06e..61a5fbc 100644 --- a/target/nextflow/dataflow/combine_samples/main.nf +++ b/target/nextflow/dataflow/combine_samples/main.nf @@ -2972,9 +2972,9 @@ meta = [ "engine" : "native|native", "output" : "target/nextflow/dataflow/combine_samples", "viash_version" : "0.9.0", - "git_commit" : "45accaa50e52c6b29c78259995293789f345c80d", - "git_remote" : "https://x-access-token:ghs_Yd3of90BXneMupSK4qIbEdGLmCGJet1c95Fo@github.com/viash-hub/demultiplex", - "git_tag" : "v0.1.1-11-g45accaa" + "git_commit" : "8d3c2888d6df6688763d303dba03ce53d339fb0f", + "git_remote" : "https://x-access-token:ghs_Iv5SuazXszUy4WWFyKOtFABImrj2oY0GsE3i@github.com/viash-hub/demultiplex", + "git_tag" : "v0.1.1-12-g8d3c288" }, "package_config" : { "name" : "demultiplex", diff --git a/target/nextflow/dataflow/gather_fastqs_and_validate/.config.vsh.yaml b/target/nextflow/dataflow/gather_fastqs_and_validate/.config.vsh.yaml index af62099..1b9fc4a 100644 --- a/target/nextflow/dataflow/gather_fastqs_and_validate/.config.vsh.yaml +++ b/target/nextflow/dataflow/gather_fastqs_and_validate/.config.vsh.yaml @@ -133,9 +133,9 @@ build_info: output: "target/nextflow/dataflow/gather_fastqs_and_validate" executable: "target/nextflow/dataflow/gather_fastqs_and_validate/main.nf" viash_version: "0.9.0" - git_commit: "45accaa50e52c6b29c78259995293789f345c80d" - git_remote: "https://x-access-token:ghs_Yd3of90BXneMupSK4qIbEdGLmCGJet1c95Fo@github.com/viash-hub/demultiplex" - git_tag: "v0.1.1-11-g45accaa" + git_commit: "8d3c2888d6df6688763d303dba03ce53d339fb0f" + git_remote: "https://x-access-token:ghs_Iv5SuazXszUy4WWFyKOtFABImrj2oY0GsE3i@github.com/viash-hub/demultiplex" + git_tag: "v0.1.1-12-g8d3c288" package_config: name: "demultiplex" version: "main" diff --git a/target/nextflow/dataflow/gather_fastqs_and_validate/main.nf b/target/nextflow/dataflow/gather_fastqs_and_validate/main.nf index 2e8061c..fb6bd13 100644 --- a/target/nextflow/dataflow/gather_fastqs_and_validate/main.nf +++ b/target/nextflow/dataflow/gather_fastqs_and_validate/main.nf @@ -2965,9 +2965,9 @@ meta = [ "engine" : "native|native", "output" : "target/nextflow/dataflow/gather_fastqs_and_validate", "viash_version" : "0.9.0", - "git_commit" : "45accaa50e52c6b29c78259995293789f345c80d", - "git_remote" : "https://x-access-token:ghs_Yd3of90BXneMupSK4qIbEdGLmCGJet1c95Fo@github.com/viash-hub/demultiplex", - "git_tag" : "v0.1.1-11-g45accaa" + "git_commit" : "8d3c2888d6df6688763d303dba03ce53d339fb0f", + "git_remote" : "https://x-access-token:ghs_Iv5SuazXszUy4WWFyKOtFABImrj2oY0GsE3i@github.com/viash-hub/demultiplex", + "git_tag" : "v0.1.1-12-g8d3c288" }, "package_config" : { "name" : "demultiplex", diff --git a/target/nextflow/demultiplex/.config.vsh.yaml b/target/nextflow/demultiplex/.config.vsh.yaml index 0ba0efe..846a585 100644 --- a/target/nextflow/demultiplex/.config.vsh.yaml +++ b/target/nextflow/demultiplex/.config.vsh.yaml @@ -220,9 +220,9 @@ build_info: output: "target/nextflow/demultiplex" executable: "target/nextflow/demultiplex/main.nf" viash_version: "0.9.0" - git_commit: "45accaa50e52c6b29c78259995293789f345c80d" - git_remote: "https://x-access-token:ghs_Yd3of90BXneMupSK4qIbEdGLmCGJet1c95Fo@github.com/viash-hub/demultiplex" - git_tag: "v0.1.1-11-g45accaa" + git_commit: "8d3c2888d6df6688763d303dba03ce53d339fb0f" + git_remote: "https://x-access-token:ghs_Iv5SuazXszUy4WWFyKOtFABImrj2oY0GsE3i@github.com/viash-hub/demultiplex" + git_tag: "v0.1.1-12-g8d3c288" dependencies: - "target/nextflow/io/untar" - "target/nextflow/dataflow/gather_fastqs_and_validate" diff --git a/target/nextflow/demultiplex/main.nf b/target/nextflow/demultiplex/main.nf index ddce380..2de6cb9 100644 --- a/target/nextflow/demultiplex/main.nf +++ b/target/nextflow/demultiplex/main.nf @@ -3088,9 +3088,9 @@ meta = [ "engine" : "native|native", "output" : "target/nextflow/demultiplex", "viash_version" : "0.9.0", - "git_commit" : "45accaa50e52c6b29c78259995293789f345c80d", - "git_remote" : "https://x-access-token:ghs_Yd3of90BXneMupSK4qIbEdGLmCGJet1c95Fo@github.com/viash-hub/demultiplex", - "git_tag" : "v0.1.1-11-g45accaa" + "git_commit" : "8d3c2888d6df6688763d303dba03ce53d339fb0f", + "git_remote" : "https://x-access-token:ghs_Iv5SuazXszUy4WWFyKOtFABImrj2oY0GsE3i@github.com/viash-hub/demultiplex", + "git_tag" : "v0.1.1-12-g8d3c288" }, "package_config" : { "name" : "demultiplex", diff --git a/target/nextflow/io/interop_summary_to_csv/.config.vsh.yaml b/target/nextflow/io/interop_summary_to_csv/.config.vsh.yaml index a481238..6b517d3 100644 --- a/target/nextflow/io/interop_summary_to_csv/.config.vsh.yaml +++ b/target/nextflow/io/interop_summary_to_csv/.config.vsh.yaml @@ -141,9 +141,9 @@ build_info: output: "target/nextflow/io/interop_summary_to_csv" executable: "target/nextflow/io/interop_summary_to_csv/main.nf" viash_version: "0.9.0" - git_commit: "45accaa50e52c6b29c78259995293789f345c80d" - git_remote: "https://x-access-token:ghs_Yd3of90BXneMupSK4qIbEdGLmCGJet1c95Fo@github.com/viash-hub/demultiplex" - git_tag: "v0.1.1-11-g45accaa" + git_commit: "8d3c2888d6df6688763d303dba03ce53d339fb0f" + git_remote: "https://x-access-token:ghs_Iv5SuazXszUy4WWFyKOtFABImrj2oY0GsE3i@github.com/viash-hub/demultiplex" + git_tag: "v0.1.1-12-g8d3c288" package_config: name: "demultiplex" version: "main" diff --git a/target/nextflow/io/interop_summary_to_csv/main.nf b/target/nextflow/io/interop_summary_to_csv/main.nf index 10d8522..4fed0dd 100644 --- a/target/nextflow/io/interop_summary_to_csv/main.nf +++ b/target/nextflow/io/interop_summary_to_csv/main.nf @@ -2977,9 +2977,9 @@ meta = [ "engine" : "docker|native", "output" : "target/nextflow/io/interop_summary_to_csv", "viash_version" : "0.9.0", - "git_commit" : "45accaa50e52c6b29c78259995293789f345c80d", - "git_remote" : "https://x-access-token:ghs_Yd3of90BXneMupSK4qIbEdGLmCGJet1c95Fo@github.com/viash-hub/demultiplex", - "git_tag" : "v0.1.1-11-g45accaa" + "git_commit" : "8d3c2888d6df6688763d303dba03ce53d339fb0f", + "git_remote" : "https://x-access-token:ghs_Iv5SuazXszUy4WWFyKOtFABImrj2oY0GsE3i@github.com/viash-hub/demultiplex", + "git_tag" : "v0.1.1-12-g8d3c288" }, "package_config" : { "name" : "demultiplex", diff --git a/target/nextflow/io/publish/.config.vsh.yaml b/target/nextflow/io/publish/.config.vsh.yaml index 823a726..a21fb44 100644 --- a/target/nextflow/io/publish/.config.vsh.yaml +++ b/target/nextflow/io/publish/.config.vsh.yaml @@ -170,9 +170,9 @@ build_info: output: "target/nextflow/io/publish" executable: "target/nextflow/io/publish/main.nf" viash_version: "0.9.0" - git_commit: "45accaa50e52c6b29c78259995293789f345c80d" - git_remote: "https://x-access-token:ghs_Yd3of90BXneMupSK4qIbEdGLmCGJet1c95Fo@github.com/viash-hub/demultiplex" - git_tag: "v0.1.1-11-g45accaa" + git_commit: "8d3c2888d6df6688763d303dba03ce53d339fb0f" + git_remote: "https://x-access-token:ghs_Iv5SuazXszUy4WWFyKOtFABImrj2oY0GsE3i@github.com/viash-hub/demultiplex" + git_tag: "v0.1.1-12-g8d3c288" package_config: name: "demultiplex" version: "main" diff --git a/target/nextflow/io/publish/main.nf b/target/nextflow/io/publish/main.nf index a27615b..dd8aa1d 100644 --- a/target/nextflow/io/publish/main.nf +++ b/target/nextflow/io/publish/main.nf @@ -3012,9 +3012,9 @@ meta = [ "engine" : "docker|native", "output" : "target/nextflow/io/publish", "viash_version" : "0.9.0", - "git_commit" : "45accaa50e52c6b29c78259995293789f345c80d", - "git_remote" : "https://x-access-token:ghs_Yd3of90BXneMupSK4qIbEdGLmCGJet1c95Fo@github.com/viash-hub/demultiplex", - "git_tag" : "v0.1.1-11-g45accaa" + "git_commit" : "8d3c2888d6df6688763d303dba03ce53d339fb0f", + "git_remote" : "https://x-access-token:ghs_Iv5SuazXszUy4WWFyKOtFABImrj2oY0GsE3i@github.com/viash-hub/demultiplex", + "git_tag" : "v0.1.1-12-g8d3c288" }, "package_config" : { "name" : "demultiplex", diff --git a/target/nextflow/io/untar/.config.vsh.yaml b/target/nextflow/io/untar/.config.vsh.yaml index 4e1673d..ccd953d 100644 --- a/target/nextflow/io/untar/.config.vsh.yaml +++ b/target/nextflow/io/untar/.config.vsh.yaml @@ -148,9 +148,9 @@ build_info: output: "target/nextflow/io/untar" executable: "target/nextflow/io/untar/main.nf" viash_version: "0.9.0" - git_commit: "45accaa50e52c6b29c78259995293789f345c80d" - git_remote: "https://x-access-token:ghs_Yd3of90BXneMupSK4qIbEdGLmCGJet1c95Fo@github.com/viash-hub/demultiplex" - git_tag: "v0.1.1-11-g45accaa" + git_commit: "8d3c2888d6df6688763d303dba03ce53d339fb0f" + git_remote: "https://x-access-token:ghs_Iv5SuazXszUy4WWFyKOtFABImrj2oY0GsE3i@github.com/viash-hub/demultiplex" + git_tag: "v0.1.1-12-g8d3c288" package_config: name: "demultiplex" version: "main" diff --git a/target/nextflow/io/untar/main.nf b/target/nextflow/io/untar/main.nf index d5bed0f..779360d 100644 --- a/target/nextflow/io/untar/main.nf +++ b/target/nextflow/io/untar/main.nf @@ -2989,9 +2989,9 @@ meta = [ "engine" : "docker|native", "output" : "target/nextflow/io/untar", "viash_version" : "0.9.0", - "git_commit" : "45accaa50e52c6b29c78259995293789f345c80d", - "git_remote" : "https://x-access-token:ghs_Yd3of90BXneMupSK4qIbEdGLmCGJet1c95Fo@github.com/viash-hub/demultiplex", - "git_tag" : "v0.1.1-11-g45accaa" + "git_commit" : "8d3c2888d6df6688763d303dba03ce53d339fb0f", + "git_remote" : "https://x-access-token:ghs_Iv5SuazXszUy4WWFyKOtFABImrj2oY0GsE3i@github.com/viash-hub/demultiplex", + "git_tag" : "v0.1.1-12-g8d3c288" }, "package_config" : { "name" : "demultiplex", diff --git a/target/nextflow/runner/.config.vsh.yaml b/target/nextflow/runner/.config.vsh.yaml index bc9fa22..32ba89d 100644 --- a/target/nextflow/runner/.config.vsh.yaml +++ b/target/nextflow/runner/.config.vsh.yaml @@ -5,7 +5,9 @@ argument_groups: arguments: - type: "file" name: "--input" - description: "Base directory of the form `s3://Sequencing///`" + description: "Base directory of the canonical form `s3://///`.\n\ + A tarball (tar.gz, .tgz, .tar) containing run information can be provided in\ + \ which\ncase the RunID is set to the name of the tarball without the extension.\n" info: null must_exist: true create_parent: true @@ -42,14 +44,9 @@ argument_groups: - name: "Annotation flags" arguments: - type: "boolean_true" - name: "--add_date_time" - description: "Add date and time to the output directory name. This is useful\n\ - when running the same pipeline multiple times on the same input\ndirectory.\n" - info: null - direction: "input" - - type: "boolean_true" - name: "--add_workflow_id" - description: "Add a workflow identifier to the output directory name.\n" + name: "--plain_output" + description: "Flag to indicate that the output should be stored directly under\ + \ $publish_dir rather than\nunder a subdirectory structure runID/_demultiplex_/.\n" info: null direction: "input" - name: "Output arguments" @@ -182,9 +179,9 @@ build_info: output: "target/nextflow/runner" executable: "target/nextflow/runner/main.nf" viash_version: "0.9.0" - git_commit: "45accaa50e52c6b29c78259995293789f345c80d" - git_remote: "https://x-access-token:ghs_Yd3of90BXneMupSK4qIbEdGLmCGJet1c95Fo@github.com/viash-hub/demultiplex" - git_tag: "v0.1.1-11-g45accaa" + git_commit: "8d3c2888d6df6688763d303dba03ce53d339fb0f" + git_remote: "https://x-access-token:ghs_Iv5SuazXszUy4WWFyKOtFABImrj2oY0GsE3i@github.com/viash-hub/demultiplex" + git_tag: "v0.1.1-12-g8d3c288" dependencies: - "target/nextflow/demultiplex" - "target/nextflow/io/publish" diff --git a/target/nextflow/runner/main.nf b/target/nextflow/runner/main.nf index 1841758..e7e928b 100644 --- a/target/nextflow/runner/main.nf +++ b/target/nextflow/runner/main.nf @@ -2813,7 +2813,7 @@ meta = [ { "type" : "file", "name" : "--input", - "description" : "Base directory of the form `s3://Sequencing///`", + "description" : "Base directory of the canonical form `s3://///`.\nA tarball (tar.gz, .tgz, .tar) containing run information can be provided in which\ncase the RunID is set to the name of the tarball without the extension.\n", "must_exist" : true, "create_parent" : true, "required" : true, @@ -2852,14 +2852,8 @@ meta = [ "arguments" : [ { "type" : "boolean_true", - "name" : "--add_date_time", - "description" : "Add date and time to the output directory name. This is useful\nwhen running the same pipeline multiple times on the same input\ndirectory.\n", - "direction" : "input" - }, - { - "type" : "boolean_true", - "name" : "--add_workflow_id", - "description" : "Add a workflow identifier to the output directory name.\n", + "name" : "--plain_output", + "description" : "Flag to indicate that the output should be stored directly under $publish_dir rather than\nunder a subdirectory structure runID/_demultiplex_/.\n", "direction" : "input" } ] @@ -3027,9 +3021,9 @@ meta = [ "engine" : "native|native", "output" : "target/nextflow/runner", "viash_version" : "0.9.0", - "git_commit" : "45accaa50e52c6b29c78259995293789f345c80d", - "git_remote" : "https://x-access-token:ghs_Yd3of90BXneMupSK4qIbEdGLmCGJet1c95Fo@github.com/viash-hub/demultiplex", - "git_tag" : "v0.1.1-11-g45accaa" + "git_commit" : "8d3c2888d6df6688763d303dba03ce53d339fb0f", + "git_remote" : "https://x-access-token:ghs_Iv5SuazXszUy4WWFyKOtFABImrj2oY0GsE3i@github.com/viash-hub/demultiplex", + "git_tag" : "v0.1.1-12-g8d3c288" }, "package_config" : { "name" : "demultiplex", @@ -3086,6 +3080,15 @@ workflow run_wf { main: output_ch = input_ch + // Extract the ID from the input. + // If the input is a tarball, strip the suffix. + | map{ id, state -> + def id_with_suffix = state.input.getFileName().toString() + [ + id, + state + [ run_id: id_with_suffix - ~/\.(tar.gz|tgz|tar)$/ ] + ] + } | demultiplex.run( fromState: [ "input": "input", @@ -3101,14 +3104,15 @@ workflow run_wf { ) | publish.run( fromState: { id, state -> - def id1 = (params.add_date_time) ? "${id}_${date}" : id - def id2 = (params.add_workflow_id) ? "${id1}_demultiplex_${version}" : id1 + println(state.plain_output) + def id1 = (state.plain_output) ? id : "${state.run_id}/${date}" + def id2 = (state.plain_output) ? id : "${id1}_demultiplex_${version}" - def fastq_output_1 = (id == "run") ? state.fastq_output : "${id2}/" + state.fastq_output - def falco_output_1 = (id == "run") ? state.falco_output : "${id2}/" + state.falco_output - def multiqc_output_1 = (id == "run") ? state.multiqc_output : "${id2}/" + state.multiqc_output + def fastq_output_1 = (id2 == "run") ? state.fastq_output : "${id2}/" + state.fastq_output + def falco_output_1 = (id2 == "run") ? state.falco_output : "${id2}/" + state.falco_output + def multiqc_output_1 = (id2 == "run") ? state.multiqc_output : "${id2}/" + state.multiqc_output - if (id == "run") { + if (id2 == "run") { println("Publising to ${params.publish_dir}") } else { println("Publising to ${params.publish_dir}/${id2}") diff --git a/target/nextflow/runner/nextflow_schema.json b/target/nextflow/runner/nextflow_schema.json index 47af365..7153e21 100644 --- a/target/nextflow/runner/nextflow_schema.json +++ b/target/nextflow/runner/nextflow_schema.json @@ -17,8 +17,8 @@ "input": { "type": "string", - "description": "Type: `file`, required. Base directory of the form `s3:/\u003cbucket\u003e/Sequencing/\u003cSequencer\u003e/\u003cRunID\u003e/`", - "help_text": "Type: `file`, required. Base directory of the form `s3:/\u003cbucket\u003e/Sequencing/\u003cSequencer\u003e/\u003cRunID\u003e/`" + "description": "Type: `file`, required. Base directory of the canonical form `s3://\u003cbucket\u003e/\u003cpath\u003e/\u003cRunID\u003e/`", + "help_text": "Type: `file`, required. Base directory of the canonical form `s3://\u003cbucket\u003e/\u003cpath\u003e/\u003cRunID\u003e/`.\nA tarball (tar.gz, .tgz, .tar) containing run information can be provided in which\ncase the RunID is set to the name of the tarball without the extension.\n" } @@ -56,22 +56,11 @@ "properties": { - "add_date_time": { + "plain_output": { "type": "boolean", - "description": "Type: `boolean_true`, default: `false`. Add date and time to the output directory name", - "help_text": "Type: `boolean_true`, default: `false`. Add date and time to the output directory name. This is useful\nwhen running the same pipeline multiple times on the same input\ndirectory.\n" - , - "default":false - } - - - , - "add_workflow_id": { - "type": - "boolean", - "description": "Type: `boolean_true`, default: `false`. Add a workflow identifier to the output directory name", - "help_text": "Type: `boolean_true`, default: `false`. Add a workflow identifier to the output directory name.\n" + "description": "Type: `boolean_true`, default: `false`. Flag to indicate that the output should be stored directly under $publish_dir rather than\nunder a subdirectory structure runID/\u003cdate_time\u003e_demultiplex_\u003cversion\u003e/", + "help_text": "Type: `boolean_true`, default: `false`. Flag to indicate that the output should be stored directly under $publish_dir rather than\nunder a subdirectory structure runID/\u003cdate_time\u003e_demultiplex_\u003cversion\u003e/.\n" , "default":false }