diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d7f413..957a9f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +# demultiplex v0.1.2 + +## Minor updates + +* Add resource labels to workflows (PR #21). + # demultiplex v0.1.1 ## Minor updates diff --git a/src/config/labels.config b/src/config/labels.config index 9ea88b1..e7ab7b8 100644 --- a/src/config/labels.config +++ b/src/config/labels.config @@ -1,26 +1,81 @@ process { - // Default resources for components that hardly do any processing - memory = { 2.GB * task.attempt } - cpus = 1 + container = 'nextflow/bash:latest' + + // default resources + memory = { 8.Gb * task.attempt } + cpus = 8 + maxForks = 36 // Retry for exit codes that have something to do with memory issues errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } maxRetries = 3 - maxMemory = null + maxMemory = 192.GB // Resource labels - withLabel: singlecpu { cpus = 1 } - withLabel: lowcpu { cpus = 4 } - withLabel: midcpu { cpus = 10 } - withLabel: highcpu { cpus = 20 } + withLabel: verylowcpu { cpus = 2 } + withLabel: lowcpu { cpus = 8 } + withLabel: midcpu { cpus = 16 } + withLabel: highcpu { cpus = 32 } - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } + withLabel: verylowmem { memory = { get_memory( 4.GB * task.attempt ) } } + withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 16.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 64.GB * task.attempt ) } } } +profiles { + // detect tempdir + tempDir = java.nio.file.Paths.get( + System.getenv('NXF_TEMP') ?: + System.getenv('VIASH_TEMP') ?: + System.getenv('TEMPDIR') ?: + System.getenv('TMPDIR') ?: + '/tmp' + ).toAbsolutePath() + + mount_temp { + docker.temp = tempDir + podman.temp = tempDir + charliecloud.temp = tempDir + } + + no_publish { + process { + withName: '.*' { + publishDir = [ + enabled: false + ] + } + } + } + + docker { + docker.fixOwnership = true + docker.enabled = true + // docker.userEmulation = true + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + + local { + // This config is for local processing. + process { + maxMemory = 25.GB + withLabel: verylowcpu { cpus = 2 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 6 } + withLabel: highcpu { cpus = 12 } + + withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 12.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 20.GB * task.attempt ) } } + } + } +} + def get_memory(to_compare) { if (!process.containsKey("maxMemory") || !process.maxMemory) { return to_compare diff --git a/src/config/tests.config b/src/config/tests.config deleted file mode 100644 index 3473a3d..0000000 --- a/src/config/tests.config +++ /dev/null @@ -1,38 +0,0 @@ - -process.container = 'nextflow/bash:latest' -profiles { - // detect tempdir - tempDir = java.nio.file.Paths.get( - System.getenv('NXF_TEMP') ?: - System.getenv('VIASH_TEMP') ?: - System.getenv('TEMPDIR') ?: - System.getenv('TMPDIR') ?: - '/tmp' - ).toAbsolutePath() - - mount_temp { - docker.temp = tempDir - podman.temp = tempDir - charliecloud.temp = tempDir - } - - no_publish { - process { - withName: '.*' { - publishDir = [ - enabled: false - ] - } - } - } - - docker { - docker.fixOwnership = true - docker.enabled = true - // docker.userEmulation = true - singularity.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false - } -} diff --git a/src/demultiplex/integration_tests.sh b/src/demultiplex/integration_tests.sh index 5d01dcd..c5f6901 100755 --- a/src/demultiplex/integration_tests.sh +++ b/src/demultiplex/integration_tests.sh @@ -10,8 +10,8 @@ viash ns build --setup cb nextflow run . \ -main-script src/demultiplex/test.nf \ - -profile docker,no_publish \ + -profile docker,no_publish,local \ -entry test_wf \ - -c src/config/tests.config \ + -c src/config/labels.config \ --resources_test https://raw.githubusercontent.com/nf-core/test-datasets/demultiplex/testdata/NovaSeq6000/ \ -resume diff --git a/src/demultiplex/main.nf b/src/demultiplex/main.nf index 4c401da..fa3c3ae 100644 --- a/src/demultiplex/main.nf +++ b/src/demultiplex/main.nf @@ -6,6 +6,7 @@ workflow run_wf { samples_ch = input_ch // untar input if needed | untar.run( + directives: [label: ["lowmem", "lowcpu"]], runIf: {id, state -> def inputStr = state.input.toString() inputStr.endsWith(".tar.gz") || \ @@ -38,6 +39,7 @@ workflow run_wf { } | interop_summary_to_csv.run( + directives: [label: ["lowmem", "verylowcpu"]], fromState: [ "input": "input", ], @@ -48,6 +50,7 @@ workflow run_wf { ) // run bcl_convert | bcl_convert.run( + directives: [label: ["highmem", "midcpu"]], fromState: [ "bcl_input_directory": "input", "sample_sheet": "sample_sheet", @@ -89,6 +92,7 @@ workflow run_wf { ] ) | falco.run( + directives: [label: ["lowcpu", "lowmem"]], fromState: {id, state -> reverse_fastqs_list = state.reverse_fastqs ? state.reverse_fastqs : [] [ @@ -104,6 +108,7 @@ workflow run_wf { }, ) | multiqc.run( + directives: [label: ["lowcpu", "lowmem"]], fromState: {id, state -> [ "input": [ diff --git a/target/executable/io/interop_summary_to_csv/.config.vsh.yaml b/target/executable/io/interop_summary_to_csv/.config.vsh.yaml index a513d54..1b05983 100644 --- a/target/executable/io/interop_summary_to_csv/.config.vsh.yaml +++ b/target/executable/io/interop_summary_to_csv/.config.vsh.yaml @@ -141,9 +141,9 @@ build_info: output: "target/executable/io/interop_summary_to_csv" executable: "target/executable/io/interop_summary_to_csv/interop_summary_to_csv" viash_version: "0.9.0" - git_commit: "a311a12056e0dbcfa8103ab0a4f6c7350f2f4c4e" - git_remote: "https://x-access-token:ghs_I5gscIGc0N3NRaJ8ljNMSCb4w56xXA2xlViM@github.com/viash-hub/demultiplex" - git_tag: "v0.1.1-2-ga311a12" + git_commit: "5cb13230bf682321226addce896a3015e8864913" + git_remote: "https://x-access-token:ghs_x4A4kChpmjRCoWA3E68PwNsGJxIsF40DBeTf@github.com/viash-hub/demultiplex" + git_tag: "v0.1.1-3-g5cb1323" package_config: name: "demultiplex" version: "main" diff --git a/target/executable/io/interop_summary_to_csv/interop_summary_to_csv b/target/executable/io/interop_summary_to_csv/interop_summary_to_csv index bbc8ad5..0abf6f6 100755 --- a/target/executable/io/interop_summary_to_csv/interop_summary_to_csv +++ b/target/executable/io/interop_summary_to_csv/interop_summary_to_csv @@ -470,9 +470,9 @@ tar -C /tmp/ --no-same-owner --no-same-permissions -xvf /tmp/interop.tar.gz && \ mv /tmp/interop-1.3.1-Linux-GNU/bin/index-summary /tmp/interop-1.3.1-Linux-GNU/bin/summary /usr/local/bin/ LABEL org.opencontainers.image.description="Companion container for running component io interop_summary_to_csv" -LABEL org.opencontainers.image.created="2024-10-08T12:19:34Z" +LABEL org.opencontainers.image.created="2024-11-06T17:42:24Z" LABEL org.opencontainers.image.source="https://github.com/viash-hub/demultiplex" -LABEL org.opencontainers.image.revision="a311a12056e0dbcfa8103ab0a4f6c7350f2f4c4e" +LABEL org.opencontainers.image.revision="5cb13230bf682321226addce896a3015e8864913" LABEL org.opencontainers.image.version="main" VIASHDOCKER diff --git a/target/executable/io/untar/.config.vsh.yaml b/target/executable/io/untar/.config.vsh.yaml index 7e5eb0b..8281ebb 100644 --- a/target/executable/io/untar/.config.vsh.yaml +++ b/target/executable/io/untar/.config.vsh.yaml @@ -148,9 +148,9 @@ build_info: output: "target/executable/io/untar" executable: "target/executable/io/untar/untar" viash_version: "0.9.0" - git_commit: "a311a12056e0dbcfa8103ab0a4f6c7350f2f4c4e" - git_remote: "https://x-access-token:ghs_I5gscIGc0N3NRaJ8ljNMSCb4w56xXA2xlViM@github.com/viash-hub/demultiplex" - git_tag: "v0.1.1-2-ga311a12" + git_commit: "5cb13230bf682321226addce896a3015e8864913" + git_remote: "https://x-access-token:ghs_x4A4kChpmjRCoWA3E68PwNsGJxIsF40DBeTf@github.com/viash-hub/demultiplex" + git_tag: "v0.1.1-3-g5cb1323" package_config: name: "demultiplex" version: "main" diff --git a/target/executable/io/untar/untar b/target/executable/io/untar/untar index 41f8df7..50cac17 100755 --- a/target/executable/io/untar/untar +++ b/target/executable/io/untar/untar @@ -476,9 +476,9 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* LABEL org.opencontainers.image.description="Companion container for running component io untar" -LABEL org.opencontainers.image.created="2024-10-08T12:19:34Z" +LABEL org.opencontainers.image.created="2024-11-06T17:42:24Z" LABEL org.opencontainers.image.source="https://github.com/viash-hub/demultiplex" -LABEL org.opencontainers.image.revision="a311a12056e0dbcfa8103ab0a4f6c7350f2f4c4e" +LABEL org.opencontainers.image.revision="5cb13230bf682321226addce896a3015e8864913" LABEL org.opencontainers.image.version="main" VIASHDOCKER diff --git a/target/nextflow/dataflow/combine_samples/.config.vsh.yaml b/target/nextflow/dataflow/combine_samples/.config.vsh.yaml index 4e428eb..24aeeda 100644 --- a/target/nextflow/dataflow/combine_samples/.config.vsh.yaml +++ b/target/nextflow/dataflow/combine_samples/.config.vsh.yaml @@ -139,9 +139,9 @@ build_info: output: "target/nextflow/dataflow/combine_samples" executable: "target/nextflow/dataflow/combine_samples/main.nf" viash_version: "0.9.0" - git_commit: "a311a12056e0dbcfa8103ab0a4f6c7350f2f4c4e" - git_remote: "https://x-access-token:ghs_I5gscIGc0N3NRaJ8ljNMSCb4w56xXA2xlViM@github.com/viash-hub/demultiplex" - git_tag: "v0.1.1-2-ga311a12" + git_commit: "5cb13230bf682321226addce896a3015e8864913" + git_remote: "https://x-access-token:ghs_x4A4kChpmjRCoWA3E68PwNsGJxIsF40DBeTf@github.com/viash-hub/demultiplex" + git_tag: "v0.1.1-3-g5cb1323" package_config: name: "demultiplex" version: "main" diff --git a/target/nextflow/dataflow/combine_samples/main.nf b/target/nextflow/dataflow/combine_samples/main.nf index 5562d8d..62438a4 100644 --- a/target/nextflow/dataflow/combine_samples/main.nf +++ b/target/nextflow/dataflow/combine_samples/main.nf @@ -2972,9 +2972,9 @@ meta = [ "engine" : "native|native", "output" : "target/nextflow/dataflow/combine_samples", "viash_version" : "0.9.0", - "git_commit" : "a311a12056e0dbcfa8103ab0a4f6c7350f2f4c4e", - "git_remote" : "https://x-access-token:ghs_I5gscIGc0N3NRaJ8ljNMSCb4w56xXA2xlViM@github.com/viash-hub/demultiplex", - "git_tag" : "v0.1.1-2-ga311a12" + "git_commit" : "5cb13230bf682321226addce896a3015e8864913", + "git_remote" : "https://x-access-token:ghs_x4A4kChpmjRCoWA3E68PwNsGJxIsF40DBeTf@github.com/viash-hub/demultiplex", + "git_tag" : "v0.1.1-3-g5cb1323" }, "package_config" : { "name" : "demultiplex", diff --git a/target/nextflow/dataflow/gather_fastqs_and_validate/.config.vsh.yaml b/target/nextflow/dataflow/gather_fastqs_and_validate/.config.vsh.yaml index 5b5c3f7..7a633ef 100644 --- a/target/nextflow/dataflow/gather_fastqs_and_validate/.config.vsh.yaml +++ b/target/nextflow/dataflow/gather_fastqs_and_validate/.config.vsh.yaml @@ -133,9 +133,9 @@ build_info: output: "target/nextflow/dataflow/gather_fastqs_and_validate" executable: "target/nextflow/dataflow/gather_fastqs_and_validate/main.nf" viash_version: "0.9.0" - git_commit: "a311a12056e0dbcfa8103ab0a4f6c7350f2f4c4e" - git_remote: "https://x-access-token:ghs_I5gscIGc0N3NRaJ8ljNMSCb4w56xXA2xlViM@github.com/viash-hub/demultiplex" - git_tag: "v0.1.1-2-ga311a12" + git_commit: "5cb13230bf682321226addce896a3015e8864913" + git_remote: "https://x-access-token:ghs_x4A4kChpmjRCoWA3E68PwNsGJxIsF40DBeTf@github.com/viash-hub/demultiplex" + git_tag: "v0.1.1-3-g5cb1323" package_config: name: "demultiplex" version: "main" diff --git a/target/nextflow/dataflow/gather_fastqs_and_validate/main.nf b/target/nextflow/dataflow/gather_fastqs_and_validate/main.nf index e7a67e6..d0085f0 100644 --- a/target/nextflow/dataflow/gather_fastqs_and_validate/main.nf +++ b/target/nextflow/dataflow/gather_fastqs_and_validate/main.nf @@ -2965,9 +2965,9 @@ meta = [ "engine" : "native|native", "output" : "target/nextflow/dataflow/gather_fastqs_and_validate", "viash_version" : "0.9.0", - "git_commit" : "a311a12056e0dbcfa8103ab0a4f6c7350f2f4c4e", - "git_remote" : "https://x-access-token:ghs_I5gscIGc0N3NRaJ8ljNMSCb4w56xXA2xlViM@github.com/viash-hub/demultiplex", - "git_tag" : "v0.1.1-2-ga311a12" + "git_commit" : "5cb13230bf682321226addce896a3015e8864913", + "git_remote" : "https://x-access-token:ghs_x4A4kChpmjRCoWA3E68PwNsGJxIsF40DBeTf@github.com/viash-hub/demultiplex", + "git_tag" : "v0.1.1-3-g5cb1323" }, "package_config" : { "name" : "demultiplex", diff --git a/target/nextflow/demultiplex/.config.vsh.yaml b/target/nextflow/demultiplex/.config.vsh.yaml index 8be1d88..c01e53e 100644 --- a/target/nextflow/demultiplex/.config.vsh.yaml +++ b/target/nextflow/demultiplex/.config.vsh.yaml @@ -186,9 +186,9 @@ build_info: output: "target/nextflow/demultiplex" executable: "target/nextflow/demultiplex/main.nf" viash_version: "0.9.0" - git_commit: "a311a12056e0dbcfa8103ab0a4f6c7350f2f4c4e" - git_remote: "https://x-access-token:ghs_I5gscIGc0N3NRaJ8ljNMSCb4w56xXA2xlViM@github.com/viash-hub/demultiplex" - git_tag: "v0.1.1-2-ga311a12" + git_commit: "5cb13230bf682321226addce896a3015e8864913" + git_remote: "https://x-access-token:ghs_x4A4kChpmjRCoWA3E68PwNsGJxIsF40DBeTf@github.com/viash-hub/demultiplex" + git_tag: "v0.1.1-3-g5cb1323" dependencies: - "target/nextflow/io/untar" - "target/nextflow/dataflow/gather_fastqs_and_validate" diff --git a/target/nextflow/demultiplex/main.nf b/target/nextflow/demultiplex/main.nf index 76b2e55..5a8cb2b 100644 --- a/target/nextflow/demultiplex/main.nf +++ b/target/nextflow/demultiplex/main.nf @@ -3049,9 +3049,9 @@ meta = [ "engine" : "native|native", "output" : "target/nextflow/demultiplex", "viash_version" : "0.9.0", - "git_commit" : "a311a12056e0dbcfa8103ab0a4f6c7350f2f4c4e", - "git_remote" : "https://x-access-token:ghs_I5gscIGc0N3NRaJ8ljNMSCb4w56xXA2xlViM@github.com/viash-hub/demultiplex", - "git_tag" : "v0.1.1-2-ga311a12" + "git_commit" : "5cb13230bf682321226addce896a3015e8864913", + "git_remote" : "https://x-access-token:ghs_x4A4kChpmjRCoWA3E68PwNsGJxIsF40DBeTf@github.com/viash-hub/demultiplex", + "git_tag" : "v0.1.1-3-g5cb1323" }, "package_config" : { "name" : "demultiplex", @@ -3110,6 +3110,7 @@ workflow run_wf { samples_ch = input_ch // untar input if needed | untar.run( + directives: [label: ["lowmem", "lowcpu"]], runIf: {id, state -> def inputStr = state.input.toString() inputStr.endsWith(".tar.gz") || \ @@ -3142,6 +3143,7 @@ workflow run_wf { } | interop_summary_to_csv.run( + directives: [label: ["lowmem", "verylowcpu"]], fromState: [ "input": "input", ], @@ -3152,6 +3154,7 @@ workflow run_wf { ) // run bcl_convert | bcl_convert.run( + directives: [label: ["highmem", "midcpu"]], fromState: [ "bcl_input_directory": "input", "sample_sheet": "sample_sheet", @@ -3193,6 +3196,7 @@ workflow run_wf { ] ) | falco.run( + directives: [label: ["lowcpu", "lowmem"]], fromState: {id, state -> reverse_fastqs_list = state.reverse_fastqs ? state.reverse_fastqs : [] [ @@ -3208,6 +3212,7 @@ workflow run_wf { }, ) | multiqc.run( + directives: [label: ["lowcpu", "lowmem"]], fromState: {id, state -> [ "input": [ diff --git a/target/nextflow/io/interop_summary_to_csv/.config.vsh.yaml b/target/nextflow/io/interop_summary_to_csv/.config.vsh.yaml index 0a9c4e8..66c35a2 100644 --- a/target/nextflow/io/interop_summary_to_csv/.config.vsh.yaml +++ b/target/nextflow/io/interop_summary_to_csv/.config.vsh.yaml @@ -141,9 +141,9 @@ build_info: output: "target/nextflow/io/interop_summary_to_csv" executable: "target/nextflow/io/interop_summary_to_csv/main.nf" viash_version: "0.9.0" - git_commit: "a311a12056e0dbcfa8103ab0a4f6c7350f2f4c4e" - git_remote: "https://x-access-token:ghs_I5gscIGc0N3NRaJ8ljNMSCb4w56xXA2xlViM@github.com/viash-hub/demultiplex" - git_tag: "v0.1.1-2-ga311a12" + git_commit: "5cb13230bf682321226addce896a3015e8864913" + git_remote: "https://x-access-token:ghs_x4A4kChpmjRCoWA3E68PwNsGJxIsF40DBeTf@github.com/viash-hub/demultiplex" + git_tag: "v0.1.1-3-g5cb1323" package_config: name: "demultiplex" version: "main" diff --git a/target/nextflow/io/interop_summary_to_csv/main.nf b/target/nextflow/io/interop_summary_to_csv/main.nf index d4e482b..13534c9 100644 --- a/target/nextflow/io/interop_summary_to_csv/main.nf +++ b/target/nextflow/io/interop_summary_to_csv/main.nf @@ -2977,9 +2977,9 @@ meta = [ "engine" : "docker|native", "output" : "target/nextflow/io/interop_summary_to_csv", "viash_version" : "0.9.0", - "git_commit" : "a311a12056e0dbcfa8103ab0a4f6c7350f2f4c4e", - "git_remote" : "https://x-access-token:ghs_I5gscIGc0N3NRaJ8ljNMSCb4w56xXA2xlViM@github.com/viash-hub/demultiplex", - "git_tag" : "v0.1.1-2-ga311a12" + "git_commit" : "5cb13230bf682321226addce896a3015e8864913", + "git_remote" : "https://x-access-token:ghs_x4A4kChpmjRCoWA3E68PwNsGJxIsF40DBeTf@github.com/viash-hub/demultiplex", + "git_tag" : "v0.1.1-3-g5cb1323" }, "package_config" : { "name" : "demultiplex", diff --git a/target/nextflow/io/untar/.config.vsh.yaml b/target/nextflow/io/untar/.config.vsh.yaml index 6c9838a..719bccb 100644 --- a/target/nextflow/io/untar/.config.vsh.yaml +++ b/target/nextflow/io/untar/.config.vsh.yaml @@ -148,9 +148,9 @@ build_info: output: "target/nextflow/io/untar" executable: "target/nextflow/io/untar/main.nf" viash_version: "0.9.0" - git_commit: "a311a12056e0dbcfa8103ab0a4f6c7350f2f4c4e" - git_remote: "https://x-access-token:ghs_I5gscIGc0N3NRaJ8ljNMSCb4w56xXA2xlViM@github.com/viash-hub/demultiplex" - git_tag: "v0.1.1-2-ga311a12" + git_commit: "5cb13230bf682321226addce896a3015e8864913" + git_remote: "https://x-access-token:ghs_x4A4kChpmjRCoWA3E68PwNsGJxIsF40DBeTf@github.com/viash-hub/demultiplex" + git_tag: "v0.1.1-3-g5cb1323" package_config: name: "demultiplex" version: "main" diff --git a/target/nextflow/io/untar/main.nf b/target/nextflow/io/untar/main.nf index fd0b271..4caa615 100644 --- a/target/nextflow/io/untar/main.nf +++ b/target/nextflow/io/untar/main.nf @@ -2989,9 +2989,9 @@ meta = [ "engine" : "docker|native", "output" : "target/nextflow/io/untar", "viash_version" : "0.9.0", - "git_commit" : "a311a12056e0dbcfa8103ab0a4f6c7350f2f4c4e", - "git_remote" : "https://x-access-token:ghs_I5gscIGc0N3NRaJ8ljNMSCb4w56xXA2xlViM@github.com/viash-hub/demultiplex", - "git_tag" : "v0.1.1-2-ga311a12" + "git_commit" : "5cb13230bf682321226addce896a3015e8864913", + "git_remote" : "https://x-access-token:ghs_x4A4kChpmjRCoWA3E68PwNsGJxIsF40DBeTf@github.com/viash-hub/demultiplex", + "git_tag" : "v0.1.1-3-g5cb1323" }, "package_config" : { "name" : "demultiplex",