diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f9d989..68ac6c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +# demultiplex v0.3.4 + +## Minor updates + +* Resource labels are now automatically included during build (PR #32). + # demultiplex v0.3.3 ## Breaking change diff --git a/_viash.yaml b/_viash.yaml index 0cb9609..506cc23 100644 --- a/_viash.yaml +++ b/_viash.yaml @@ -1,5 +1,5 @@ name: demultiplex -version: v0.3.3 +version: v0.3.4 description: | Demultiplexing pipeline license: MIT @@ -17,3 +17,5 @@ viash_version: 0.9.0 config_mods: | .requirements.commands := ['ps'] .runners[.type == 'nextflow'].directives.tag := '$id' + .resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'} + .runners[.type == 'nextflow'].config.script := 'includeConfig("nextflow_labels.config")' diff --git a/target/executable/io/interop_summary_to_csv/.config.vsh.yaml b/target/executable/io/interop_summary_to_csv/.config.vsh.yaml index e4ace00..9d115c2 100644 --- a/target/executable/io/interop_summary_to_csv/.config.vsh.yaml +++ b/target/executable/io/interop_summary_to_csv/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "interop_summary_to_csv" namespace: "io" -version: "v0.3.3" +version: "v0.3.4" argument_groups: - name: "Input arguments" arguments: @@ -38,6 +38,9 @@ resources: - type: "bash_script" path: "script.sh" is_executable: true +- type: "file" + path: "nextflow_labels.config" + dest: "nextflow_labels.config" info: null status: "enabled" requirements: @@ -109,6 +112,8 @@ runners: cpu200: "cpus = 200" cpu500: "cpus = 500" cpu1000: "cpus = 1000" + script: + - "includeConfig(\"nextflow_labels.config\")" debug: false container: "docker" engines: @@ -116,7 +121,7 @@ engines: id: "docker" image: "debian:stable-slim" target_registry: "images.viash-hub.com" - target_tag: "v0.3.3" + target_tag: "v0.3.4" namespace_separator: "/" setup: - type: "apt" @@ -141,12 +146,12 @@ build_info: output: "target/executable/io/interop_summary_to_csv" executable: "target/executable/io/interop_summary_to_csv/interop_summary_to_csv" viash_version: "0.9.0" - git_commit: "e297e8ff45464065384476fe521ab2700ec3739c" - git_remote: "https://x-access-token:ghs_44flLTQIYGBkc9xcwCZoLYe7mxXzgw1iszFO@github.com/viash-hub/demultiplex" - git_tag: "v0.3.2-4-ge297e8f" + git_commit: "e6f8b20d70be8f47907c08cc3c7802920cc0eee4" + git_remote: "https://x-access-token:ghs_FhW4M4TmdkVUw8Om3VZ1e7pO685xf7028pZ3@github.com/viash-hub/demultiplex" + git_tag: "v0.3.3-3-ge6f8b20" package_config: name: "demultiplex" - version: "v0.3.3" + version: "v0.3.4" description: "Demultiplexing pipeline\n" info: test_resources: @@ -157,10 +162,12 @@ package_config: target: "target" config_mods: - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\ - \ := '$id'\n" + \ := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n\ + .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ + )'\n" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.3.3'" + - ".engines[.type == 'docker'].target_tag := 'v0.3.4'" keywords: - "bioinformatics" - "sequence" diff --git a/target/executable/io/interop_summary_to_csv/interop_summary_to_csv b/target/executable/io/interop_summary_to_csv/interop_summary_to_csv index bfe2823..20f368c 100755 --- a/target/executable/io/interop_summary_to_csv/interop_summary_to_csv +++ b/target/executable/io/interop_summary_to_csv/interop_summary_to_csv @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# interop_summary_to_csv v0.3.3 +# interop_summary_to_csv v0.3.4 # # This wrapper script is auto-generated by viash 0.9.0 and is thus a derivative # work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -171,7 +171,7 @@ VIASH_META_TEMP_DIR="$VIASH_TEMP" # ViashHelp: Display helpful explanation about this executable function ViashHelp { - echo "interop_summary_to_csv v0.3.3" + echo "interop_summary_to_csv v0.3.4" echo "" echo "Input arguments:" echo " --input" @@ -470,10 +470,10 @@ tar -C /tmp/ --no-same-owner --no-same-permissions -xvf /tmp/interop.tar.gz && \ mv /tmp/interop-1.3.1-Linux-GNU/bin/index-summary /tmp/interop-1.3.1-Linux-GNU/bin/summary /usr/local/bin/ LABEL org.opencontainers.image.description="Companion container for running component io interop_summary_to_csv" -LABEL org.opencontainers.image.created="2024-12-19T15:42:23Z" +LABEL org.opencontainers.image.created="2024-12-20T12:37:34Z" LABEL org.opencontainers.image.source="https://github.com/viash-hub/demultiplex" -LABEL org.opencontainers.image.revision="e297e8ff45464065384476fe521ab2700ec3739c" -LABEL org.opencontainers.image.version="v0.3.3" +LABEL org.opencontainers.image.revision="e6f8b20d70be8f47907c08cc3c7802920cc0eee4" +LABEL org.opencontainers.image.version="v0.3.4" VIASHDOCKER fi @@ -609,7 +609,7 @@ while [[ $# -gt 0 ]]; do shift 1 ;; --version) - echo "interop_summary_to_csv v0.3.3" + echo "interop_summary_to_csv v0.3.4" exit ;; --input) @@ -733,7 +733,7 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then # determine docker image id if [[ "$VIASH_ENGINE_ID" == 'docker' ]]; then - VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/demultiplex/io/interop_summary_to_csv:v0.3.3' + VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/demultiplex/io/interop_summary_to_csv:v0.3.4' fi # print dockerfile diff --git a/target/executable/io/interop_summary_to_csv/nextflow_labels.config b/target/executable/io/interop_summary_to_csv/nextflow_labels.config new file mode 100644 index 0000000..e7ab7b8 --- /dev/null +++ b/target/executable/io/interop_summary_to_csv/nextflow_labels.config @@ -0,0 +1,98 @@ +process { + container = 'nextflow/bash:latest' + + // default resources + memory = { 8.Gb * task.attempt } + cpus = 8 + maxForks = 36 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + maxMemory = 192.GB + + // Resource labels + withLabel: verylowcpu { cpus = 2 } + withLabel: lowcpu { cpus = 8 } + withLabel: midcpu { cpus = 16 } + withLabel: highcpu { cpus = 32 } + + withLabel: verylowmem { memory = { get_memory( 4.GB * task.attempt ) } } + withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 16.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 64.GB * task.attempt ) } } + +} + +profiles { + // detect tempdir + tempDir = java.nio.file.Paths.get( + System.getenv('NXF_TEMP') ?: + System.getenv('VIASH_TEMP') ?: + System.getenv('TEMPDIR') ?: + System.getenv('TMPDIR') ?: + '/tmp' + ).toAbsolutePath() + + mount_temp { + docker.temp = tempDir + podman.temp = tempDir + charliecloud.temp = tempDir + } + + no_publish { + process { + withName: '.*' { + publishDir = [ + enabled: false + ] + } + } + } + + docker { + docker.fixOwnership = true + docker.enabled = true + // docker.userEmulation = true + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + + local { + // This config is for local processing. + process { + maxMemory = 25.GB + withLabel: verylowcpu { cpus = 2 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 6 } + withLabel: highcpu { cpus = 12 } + + withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 12.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 20.GB * task.attempt ) } } + } + } +} + +def get_memory(to_compare) { + if (!process.containsKey("maxMemory") || !process.maxMemory) { + return to_compare + } + + try { + if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { + return process.maxMemory + } + else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { + return max_memory as nextflow.util.MemoryUnit + } + else { + return to_compare + } + } catch (all) { + println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" + System.exit(1) + } + } diff --git a/target/executable/io/publish/.config.vsh.yaml b/target/executable/io/publish/.config.vsh.yaml index d89e66f..bcfda71 100644 --- a/target/executable/io/publish/.config.vsh.yaml +++ b/target/executable/io/publish/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "publish" namespace: "io" -version: "v0.3.3" +version: "v0.3.4" argument_groups: - name: "Input arguments" arguments: @@ -94,6 +94,9 @@ resources: - type: "bash_script" path: "code.sh" is_executable: true +- type: "file" + path: "nextflow_labels.config" + dest: "nextflow_labels.config" description: "Publish the processed results of the run" info: null status: "enabled" @@ -166,6 +169,8 @@ runners: cpu200: "cpus = 200" cpu500: "cpus = 500" cpu1000: "cpus = 1000" + script: + - "includeConfig(\"nextflow_labels.config\")" debug: false container: "docker" engines: @@ -173,7 +178,7 @@ engines: id: "docker" image: "debian:stable-slim" target_registry: "images.viash-hub.com" - target_tag: "v0.3.3" + target_tag: "v0.3.4" namespace_separator: "/" setup: - type: "apt" @@ -191,12 +196,12 @@ build_info: output: "target/executable/io/publish" executable: "target/executable/io/publish/publish" viash_version: "0.9.0" - git_commit: "e297e8ff45464065384476fe521ab2700ec3739c" - git_remote: "https://x-access-token:ghs_44flLTQIYGBkc9xcwCZoLYe7mxXzgw1iszFO@github.com/viash-hub/demultiplex" - git_tag: "v0.3.2-4-ge297e8f" + git_commit: "e6f8b20d70be8f47907c08cc3c7802920cc0eee4" + git_remote: "https://x-access-token:ghs_FhW4M4TmdkVUw8Om3VZ1e7pO685xf7028pZ3@github.com/viash-hub/demultiplex" + git_tag: "v0.3.3-3-ge6f8b20" package_config: name: "demultiplex" - version: "v0.3.3" + version: "v0.3.4" description: "Demultiplexing pipeline\n" info: test_resources: @@ -207,10 +212,12 @@ package_config: target: "target" config_mods: - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\ - \ := '$id'\n" + \ := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n\ + .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ + )'\n" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.3.3'" + - ".engines[.type == 'docker'].target_tag := 'v0.3.4'" keywords: - "bioinformatics" - "sequence" diff --git a/target/executable/io/publish/nextflow_labels.config b/target/executable/io/publish/nextflow_labels.config new file mode 100644 index 0000000..e7ab7b8 --- /dev/null +++ b/target/executable/io/publish/nextflow_labels.config @@ -0,0 +1,98 @@ +process { + container = 'nextflow/bash:latest' + + // default resources + memory = { 8.Gb * task.attempt } + cpus = 8 + maxForks = 36 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + maxMemory = 192.GB + + // Resource labels + withLabel: verylowcpu { cpus = 2 } + withLabel: lowcpu { cpus = 8 } + withLabel: midcpu { cpus = 16 } + withLabel: highcpu { cpus = 32 } + + withLabel: verylowmem { memory = { get_memory( 4.GB * task.attempt ) } } + withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 16.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 64.GB * task.attempt ) } } + +} + +profiles { + // detect tempdir + tempDir = java.nio.file.Paths.get( + System.getenv('NXF_TEMP') ?: + System.getenv('VIASH_TEMP') ?: + System.getenv('TEMPDIR') ?: + System.getenv('TMPDIR') ?: + '/tmp' + ).toAbsolutePath() + + mount_temp { + docker.temp = tempDir + podman.temp = tempDir + charliecloud.temp = tempDir + } + + no_publish { + process { + withName: '.*' { + publishDir = [ + enabled: false + ] + } + } + } + + docker { + docker.fixOwnership = true + docker.enabled = true + // docker.userEmulation = true + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + + local { + // This config is for local processing. + process { + maxMemory = 25.GB + withLabel: verylowcpu { cpus = 2 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 6 } + withLabel: highcpu { cpus = 12 } + + withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 12.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 20.GB * task.attempt ) } } + } + } +} + +def get_memory(to_compare) { + if (!process.containsKey("maxMemory") || !process.maxMemory) { + return to_compare + } + + try { + if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { + return process.maxMemory + } + else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { + return max_memory as nextflow.util.MemoryUnit + } + else { + return to_compare + } + } catch (all) { + println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" + System.exit(1) + } + } diff --git a/target/executable/io/publish/publish b/target/executable/io/publish/publish index 9ac5fc3..2889720 100755 --- a/target/executable/io/publish/publish +++ b/target/executable/io/publish/publish @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# publish v0.3.3 +# publish v0.3.4 # # This wrapper script is auto-generated by viash 0.9.0 and is thus a derivative # work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -171,7 +171,7 @@ VIASH_META_TEMP_DIR="$VIASH_TEMP" # ViashHelp: Display helpful explanation about this executable function ViashHelp { - echo "publish v0.3.3" + echo "publish v0.3.4" echo "" echo "Publish the processed results of the run" echo "" @@ -490,10 +490,10 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* LABEL org.opencontainers.image.description="Companion container for running component io publish" -LABEL org.opencontainers.image.created="2024-12-19T15:42:24Z" +LABEL org.opencontainers.image.created="2024-12-20T12:37:34Z" LABEL org.opencontainers.image.source="https://github.com/viash-hub/demultiplex" -LABEL org.opencontainers.image.revision="e297e8ff45464065384476fe521ab2700ec3739c" -LABEL org.opencontainers.image.version="v0.3.3" +LABEL org.opencontainers.image.revision="e6f8b20d70be8f47907c08cc3c7802920cc0eee4" +LABEL org.opencontainers.image.version="v0.3.4" VIASHDOCKER fi @@ -629,7 +629,7 @@ while [[ $# -gt 0 ]]; do shift 1 ;; --version) - echo "publish v0.3.3" + echo "publish v0.3.4" exit ;; --input) @@ -808,7 +808,7 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then # determine docker image id if [[ "$VIASH_ENGINE_ID" == 'docker' ]]; then - VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/demultiplex/io/publish:v0.3.3' + VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/demultiplex/io/publish:v0.3.4' fi # print dockerfile diff --git a/target/executable/io/untar/.config.vsh.yaml b/target/executable/io/untar/.config.vsh.yaml index 3b3d8a0..886ea7d 100644 --- a/target/executable/io/untar/.config.vsh.yaml +++ b/target/executable/io/untar/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "untar" namespace: "io" -version: "v0.3.3" +version: "v0.3.4" argument_groups: - name: "Input arguments" arguments: @@ -45,6 +45,9 @@ resources: - type: "bash_script" path: "script.sh" is_executable: true +- type: "file" + path: "nextflow_labels.config" + dest: "nextflow_labels.config" description: "Unpack a .tar file. When the contents of the .tar file is just a single\ \ directory,\nput the contents of the directory into the output folder instead of\ \ that directory.\n" @@ -123,6 +126,8 @@ runners: cpu200: "cpus = 200" cpu500: "cpus = 500" cpu1000: "cpus = 1000" + script: + - "includeConfig(\"nextflow_labels.config\")" debug: false container: "docker" engines: @@ -130,7 +135,7 @@ engines: id: "docker" image: "debian:stable-slim" target_registry: "images.viash-hub.com" - target_tag: "v0.3.3" + target_tag: "v0.3.4" namespace_separator: "/" setup: - type: "apt" @@ -148,12 +153,12 @@ build_info: output: "target/executable/io/untar" executable: "target/executable/io/untar/untar" viash_version: "0.9.0" - git_commit: "e297e8ff45464065384476fe521ab2700ec3739c" - git_remote: "https://x-access-token:ghs_44flLTQIYGBkc9xcwCZoLYe7mxXzgw1iszFO@github.com/viash-hub/demultiplex" - git_tag: "v0.3.2-4-ge297e8f" + git_commit: "e6f8b20d70be8f47907c08cc3c7802920cc0eee4" + git_remote: "https://x-access-token:ghs_FhW4M4TmdkVUw8Om3VZ1e7pO685xf7028pZ3@github.com/viash-hub/demultiplex" + git_tag: "v0.3.3-3-ge6f8b20" package_config: name: "demultiplex" - version: "v0.3.3" + version: "v0.3.4" description: "Demultiplexing pipeline\n" info: test_resources: @@ -164,10 +169,12 @@ package_config: target: "target" config_mods: - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\ - \ := '$id'\n" + \ := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n\ + .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ + )'\n" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.3.3'" + - ".engines[.type == 'docker'].target_tag := 'v0.3.4'" keywords: - "bioinformatics" - "sequence" diff --git a/target/executable/io/untar/nextflow_labels.config b/target/executable/io/untar/nextflow_labels.config new file mode 100644 index 0000000..e7ab7b8 --- /dev/null +++ b/target/executable/io/untar/nextflow_labels.config @@ -0,0 +1,98 @@ +process { + container = 'nextflow/bash:latest' + + // default resources + memory = { 8.Gb * task.attempt } + cpus = 8 + maxForks = 36 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + maxMemory = 192.GB + + // Resource labels + withLabel: verylowcpu { cpus = 2 } + withLabel: lowcpu { cpus = 8 } + withLabel: midcpu { cpus = 16 } + withLabel: highcpu { cpus = 32 } + + withLabel: verylowmem { memory = { get_memory( 4.GB * task.attempt ) } } + withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 16.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 64.GB * task.attempt ) } } + +} + +profiles { + // detect tempdir + tempDir = java.nio.file.Paths.get( + System.getenv('NXF_TEMP') ?: + System.getenv('VIASH_TEMP') ?: + System.getenv('TEMPDIR') ?: + System.getenv('TMPDIR') ?: + '/tmp' + ).toAbsolutePath() + + mount_temp { + docker.temp = tempDir + podman.temp = tempDir + charliecloud.temp = tempDir + } + + no_publish { + process { + withName: '.*' { + publishDir = [ + enabled: false + ] + } + } + } + + docker { + docker.fixOwnership = true + docker.enabled = true + // docker.userEmulation = true + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + + local { + // This config is for local processing. + process { + maxMemory = 25.GB + withLabel: verylowcpu { cpus = 2 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 6 } + withLabel: highcpu { cpus = 12 } + + withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 12.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 20.GB * task.attempt ) } } + } + } +} + +def get_memory(to_compare) { + if (!process.containsKey("maxMemory") || !process.maxMemory) { + return to_compare + } + + try { + if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { + return process.maxMemory + } + else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { + return max_memory as nextflow.util.MemoryUnit + } + else { + return to_compare + } + } catch (all) { + println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" + System.exit(1) + } + } diff --git a/target/executable/io/untar/untar b/target/executable/io/untar/untar index a74f7a7..fa24fea 100755 --- a/target/executable/io/untar/untar +++ b/target/executable/io/untar/untar @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# untar v0.3.3 +# untar v0.3.4 # # This wrapper script is auto-generated by viash 0.9.0 and is thus a derivative # work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -171,7 +171,7 @@ VIASH_META_TEMP_DIR="$VIASH_TEMP" # ViashHelp: Display helpful explanation about this executable function ViashHelp { - echo "untar v0.3.3" + echo "untar v0.3.4" echo "" echo "Unpack a .tar file. When the contents of the .tar file is just a single" echo "directory," @@ -476,10 +476,10 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* LABEL org.opencontainers.image.description="Companion container for running component io untar" -LABEL org.opencontainers.image.created="2024-12-19T15:42:24Z" +LABEL org.opencontainers.image.created="2024-12-20T12:37:34Z" LABEL org.opencontainers.image.source="https://github.com/viash-hub/demultiplex" -LABEL org.opencontainers.image.revision="e297e8ff45464065384476fe521ab2700ec3739c" -LABEL org.opencontainers.image.version="v0.3.3" +LABEL org.opencontainers.image.revision="e6f8b20d70be8f47907c08cc3c7802920cc0eee4" +LABEL org.opencontainers.image.version="v0.3.4" VIASHDOCKER fi @@ -615,7 +615,7 @@ while [[ $# -gt 0 ]]; do shift 1 ;; --version) - echo "untar v0.3.3" + echo "untar v0.3.4" exit ;; --input) @@ -745,7 +745,7 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then # determine docker image id if [[ "$VIASH_ENGINE_ID" == 'docker' ]]; then - VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/demultiplex/io/untar:v0.3.3' + VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/demultiplex/io/untar:v0.3.4' fi # print dockerfile diff --git a/target/nextflow/dataflow/combine_samples/.config.vsh.yaml b/target/nextflow/dataflow/combine_samples/.config.vsh.yaml index 381c030..5395145 100644 --- a/target/nextflow/dataflow/combine_samples/.config.vsh.yaml +++ b/target/nextflow/dataflow/combine_samples/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "combine_samples" namespace: "dataflow" -version: "v0.3.3" +version: "v0.3.4" argument_groups: - name: "Input arguments" arguments: @@ -55,6 +55,9 @@ resources: path: "main.nf" is_executable: true entrypoint: "run_wf" +- type: "file" + path: "nextflow_labels.config" + dest: "nextflow_labels.config" description: "Combine fastq files from across samples into one event with a list of\ \ fastq files per orientation." info: null @@ -125,6 +128,8 @@ runners: cpu200: "cpus = 200" cpu500: "cpus = 500" cpu1000: "cpus = 1000" + script: + - "includeConfig(\"nextflow_labels.config\")" debug: false container: "docker" engines: @@ -139,12 +144,12 @@ build_info: output: "target/nextflow/dataflow/combine_samples" executable: "target/nextflow/dataflow/combine_samples/main.nf" viash_version: "0.9.0" - git_commit: "e297e8ff45464065384476fe521ab2700ec3739c" - git_remote: "https://x-access-token:ghs_44flLTQIYGBkc9xcwCZoLYe7mxXzgw1iszFO@github.com/viash-hub/demultiplex" - git_tag: "v0.3.2-4-ge297e8f" + git_commit: "e6f8b20d70be8f47907c08cc3c7802920cc0eee4" + git_remote: "https://x-access-token:ghs_FhW4M4TmdkVUw8Om3VZ1e7pO685xf7028pZ3@github.com/viash-hub/demultiplex" + git_tag: "v0.3.3-3-ge6f8b20" package_config: name: "demultiplex" - version: "v0.3.3" + version: "v0.3.4" description: "Demultiplexing pipeline\n" info: test_resources: @@ -155,10 +160,12 @@ package_config: target: "target" config_mods: - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\ - \ := '$id'\n" + \ := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n\ + .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ + )'\n" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.3.3'" + - ".engines[.type == 'docker'].target_tag := 'v0.3.4'" keywords: - "bioinformatics" - "sequence" diff --git a/target/nextflow/dataflow/combine_samples/main.nf b/target/nextflow/dataflow/combine_samples/main.nf index f7575af..04a41c3 100644 --- a/target/nextflow/dataflow/combine_samples/main.nf +++ b/target/nextflow/dataflow/combine_samples/main.nf @@ -1,4 +1,4 @@ -// combine_samples v0.3.3 +// combine_samples v0.3.4 // // This wrapper script is auto-generated by viash 0.9.0 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -2806,7 +2806,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "combine_samples", "namespace" : "dataflow", - "version" : "v0.3.3", + "version" : "v0.3.4", "argument_groups" : [ { "name" : "Input arguments", @@ -2874,6 +2874,11 @@ meta = [ "path" : "main.nf", "is_executable" : true, "entrypoint" : "run_wf" + }, + { + "type" : "file", + "path" : "/src/config/labels.config", + "dest" : "nextflow_labels.config" } ], "description" : "Combine fastq files from across samples into one event with a list of fastq files per orientation.", @@ -2950,7 +2955,10 @@ meta = [ "cpu200" : "cpus = 200", "cpu500" : "cpus = 500", "cpu1000" : "cpus = 1000" - } + }, + "script" : [ + "includeConfig(\\"nextflow_labels.config\\")" + ] }, "debug" : false, "container" : "docker" @@ -2972,13 +2980,13 @@ meta = [ "engine" : "native|native", "output" : "target/nextflow/dataflow/combine_samples", "viash_version" : "0.9.0", - "git_commit" : "e297e8ff45464065384476fe521ab2700ec3739c", - "git_remote" : "https://x-access-token:ghs_44flLTQIYGBkc9xcwCZoLYe7mxXzgw1iszFO@github.com/viash-hub/demultiplex", - "git_tag" : "v0.3.2-4-ge297e8f" + "git_commit" : "e6f8b20d70be8f47907c08cc3c7802920cc0eee4", + "git_remote" : "https://x-access-token:ghs_FhW4M4TmdkVUw8Om3VZ1e7pO685xf7028pZ3@github.com/viash-hub/demultiplex", + "git_tag" : "v0.3.3-3-ge6f8b20" }, "package_config" : { "name" : "demultiplex", - "version" : "v0.3.3", + "version" : "v0.3.4", "description" : "Demultiplexing pipeline\n", "info" : { "test_resources" : [ @@ -2992,10 +3000,10 @@ meta = [ "source" : "src", "target" : "target", "config_mods" : [ - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n", + ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v0.3.3'" + ".engines[.type == 'docker'].target_tag := 'v0.3.4'" ], "keywords" : [ "bioinformatics", diff --git a/target/nextflow/dataflow/combine_samples/nextflow.config b/target/nextflow/dataflow/combine_samples/nextflow.config index 37d46e8..c23f949 100644 --- a/target/nextflow/dataflow/combine_samples/nextflow.config +++ b/target/nextflow/dataflow/combine_samples/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'dataflow/combine_samples' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v0.3.3' + version = 'v0.3.4' description = 'Combine fastq files from across samples into one event with a list of fastq files per orientation.' } @@ -122,4 +122,4 @@ process{ withLabel: cpu1000 { cpus = 1000 } } - +includeConfig("nextflow_labels.config") diff --git a/target/nextflow/dataflow/combine_samples/nextflow_labels.config b/target/nextflow/dataflow/combine_samples/nextflow_labels.config new file mode 100644 index 0000000..e7ab7b8 --- /dev/null +++ b/target/nextflow/dataflow/combine_samples/nextflow_labels.config @@ -0,0 +1,98 @@ +process { + container = 'nextflow/bash:latest' + + // default resources + memory = { 8.Gb * task.attempt } + cpus = 8 + maxForks = 36 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + maxMemory = 192.GB + + // Resource labels + withLabel: verylowcpu { cpus = 2 } + withLabel: lowcpu { cpus = 8 } + withLabel: midcpu { cpus = 16 } + withLabel: highcpu { cpus = 32 } + + withLabel: verylowmem { memory = { get_memory( 4.GB * task.attempt ) } } + withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 16.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 64.GB * task.attempt ) } } + +} + +profiles { + // detect tempdir + tempDir = java.nio.file.Paths.get( + System.getenv('NXF_TEMP') ?: + System.getenv('VIASH_TEMP') ?: + System.getenv('TEMPDIR') ?: + System.getenv('TMPDIR') ?: + '/tmp' + ).toAbsolutePath() + + mount_temp { + docker.temp = tempDir + podman.temp = tempDir + charliecloud.temp = tempDir + } + + no_publish { + process { + withName: '.*' { + publishDir = [ + enabled: false + ] + } + } + } + + docker { + docker.fixOwnership = true + docker.enabled = true + // docker.userEmulation = true + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + + local { + // This config is for local processing. + process { + maxMemory = 25.GB + withLabel: verylowcpu { cpus = 2 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 6 } + withLabel: highcpu { cpus = 12 } + + withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 12.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 20.GB * task.attempt ) } } + } + } +} + +def get_memory(to_compare) { + if (!process.containsKey("maxMemory") || !process.maxMemory) { + return to_compare + } + + try { + if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { + return process.maxMemory + } + else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { + return max_memory as nextflow.util.MemoryUnit + } + else { + return to_compare + } + } catch (all) { + println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" + System.exit(1) + } + } diff --git a/target/nextflow/dataflow/gather_fastqs_and_validate/.config.vsh.yaml b/target/nextflow/dataflow/gather_fastqs_and_validate/.config.vsh.yaml index 8a63229..a0a1d5e 100644 --- a/target/nextflow/dataflow/gather_fastqs_and_validate/.config.vsh.yaml +++ b/target/nextflow/dataflow/gather_fastqs_and_validate/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "gather_fastqs_and_validate" namespace: "dataflow" -version: "v0.3.3" +version: "v0.3.4" argument_groups: - name: "Input arguments" arguments: @@ -49,6 +49,9 @@ resources: path: "main.nf" is_executable: true entrypoint: "run_wf" +- type: "file" + path: "nextflow_labels.config" + dest: "nextflow_labels.config" description: "From a directory containing fastq files, gather the files per sample\ \ \nand validate according to the contents of the sample sheet.\n" info: null @@ -119,6 +122,8 @@ runners: cpu200: "cpus = 200" cpu500: "cpus = 500" cpu1000: "cpus = 1000" + script: + - "includeConfig(\"nextflow_labels.config\")" debug: false container: "docker" engines: @@ -133,12 +138,12 @@ build_info: output: "target/nextflow/dataflow/gather_fastqs_and_validate" executable: "target/nextflow/dataflow/gather_fastqs_and_validate/main.nf" viash_version: "0.9.0" - git_commit: "e297e8ff45464065384476fe521ab2700ec3739c" - git_remote: "https://x-access-token:ghs_44flLTQIYGBkc9xcwCZoLYe7mxXzgw1iszFO@github.com/viash-hub/demultiplex" - git_tag: "v0.3.2-4-ge297e8f" + git_commit: "e6f8b20d70be8f47907c08cc3c7802920cc0eee4" + git_remote: "https://x-access-token:ghs_FhW4M4TmdkVUw8Om3VZ1e7pO685xf7028pZ3@github.com/viash-hub/demultiplex" + git_tag: "v0.3.3-3-ge6f8b20" package_config: name: "demultiplex" - version: "v0.3.3" + version: "v0.3.4" description: "Demultiplexing pipeline\n" info: test_resources: @@ -149,10 +154,12 @@ package_config: target: "target" config_mods: - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\ - \ := '$id'\n" + \ := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n\ + .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ + )'\n" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.3.3'" + - ".engines[.type == 'docker'].target_tag := 'v0.3.4'" keywords: - "bioinformatics" - "sequence" diff --git a/target/nextflow/dataflow/gather_fastqs_and_validate/main.nf b/target/nextflow/dataflow/gather_fastqs_and_validate/main.nf index 83cd959..c34b193 100644 --- a/target/nextflow/dataflow/gather_fastqs_and_validate/main.nf +++ b/target/nextflow/dataflow/gather_fastqs_and_validate/main.nf @@ -1,4 +1,4 @@ -// gather_fastqs_and_validate v0.3.3 +// gather_fastqs_and_validate v0.3.4 // // This wrapper script is auto-generated by viash 0.9.0 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -2806,7 +2806,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "gather_fastqs_and_validate", "namespace" : "dataflow", - "version" : "v0.3.3", + "version" : "v0.3.4", "argument_groups" : [ { "name" : "Input arguments", @@ -2867,6 +2867,11 @@ meta = [ "path" : "main.nf", "is_executable" : true, "entrypoint" : "run_wf" + }, + { + "type" : "file", + "path" : "/src/config/labels.config", + "dest" : "nextflow_labels.config" } ], "description" : "From a directory containing fastq files, gather the files per sample \nand validate according to the contents of the sample sheet.\n", @@ -2943,7 +2948,10 @@ meta = [ "cpu200" : "cpus = 200", "cpu500" : "cpus = 500", "cpu1000" : "cpus = 1000" - } + }, + "script" : [ + "includeConfig(\\"nextflow_labels.config\\")" + ] }, "debug" : false, "container" : "docker" @@ -2965,13 +2973,13 @@ meta = [ "engine" : "native|native", "output" : "target/nextflow/dataflow/gather_fastqs_and_validate", "viash_version" : "0.9.0", - "git_commit" : "e297e8ff45464065384476fe521ab2700ec3739c", - "git_remote" : "https://x-access-token:ghs_44flLTQIYGBkc9xcwCZoLYe7mxXzgw1iszFO@github.com/viash-hub/demultiplex", - "git_tag" : "v0.3.2-4-ge297e8f" + "git_commit" : "e6f8b20d70be8f47907c08cc3c7802920cc0eee4", + "git_remote" : "https://x-access-token:ghs_FhW4M4TmdkVUw8Om3VZ1e7pO685xf7028pZ3@github.com/viash-hub/demultiplex", + "git_tag" : "v0.3.3-3-ge6f8b20" }, "package_config" : { "name" : "demultiplex", - "version" : "v0.3.3", + "version" : "v0.3.4", "description" : "Demultiplexing pipeline\n", "info" : { "test_resources" : [ @@ -2985,10 +2993,10 @@ meta = [ "source" : "src", "target" : "target", "config_mods" : [ - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n", + ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v0.3.3'" + ".engines[.type == 'docker'].target_tag := 'v0.3.4'" ], "keywords" : [ "bioinformatics", diff --git a/target/nextflow/dataflow/gather_fastqs_and_validate/nextflow.config b/target/nextflow/dataflow/gather_fastqs_and_validate/nextflow.config index 3dca43a..3a2cc35 100644 --- a/target/nextflow/dataflow/gather_fastqs_and_validate/nextflow.config +++ b/target/nextflow/dataflow/gather_fastqs_and_validate/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'dataflow/gather_fastqs_and_validate' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v0.3.3' + version = 'v0.3.4' description = 'From a directory containing fastq files, gather the files per sample \nand validate according to the contents of the sample sheet.\n' } @@ -122,4 +122,4 @@ process{ withLabel: cpu1000 { cpus = 1000 } } - +includeConfig("nextflow_labels.config") diff --git a/target/nextflow/dataflow/gather_fastqs_and_validate/nextflow_labels.config b/target/nextflow/dataflow/gather_fastqs_and_validate/nextflow_labels.config new file mode 100644 index 0000000..e7ab7b8 --- /dev/null +++ b/target/nextflow/dataflow/gather_fastqs_and_validate/nextflow_labels.config @@ -0,0 +1,98 @@ +process { + container = 'nextflow/bash:latest' + + // default resources + memory = { 8.Gb * task.attempt } + cpus = 8 + maxForks = 36 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + maxMemory = 192.GB + + // Resource labels + withLabel: verylowcpu { cpus = 2 } + withLabel: lowcpu { cpus = 8 } + withLabel: midcpu { cpus = 16 } + withLabel: highcpu { cpus = 32 } + + withLabel: verylowmem { memory = { get_memory( 4.GB * task.attempt ) } } + withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 16.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 64.GB * task.attempt ) } } + +} + +profiles { + // detect tempdir + tempDir = java.nio.file.Paths.get( + System.getenv('NXF_TEMP') ?: + System.getenv('VIASH_TEMP') ?: + System.getenv('TEMPDIR') ?: + System.getenv('TMPDIR') ?: + '/tmp' + ).toAbsolutePath() + + mount_temp { + docker.temp = tempDir + podman.temp = tempDir + charliecloud.temp = tempDir + } + + no_publish { + process { + withName: '.*' { + publishDir = [ + enabled: false + ] + } + } + } + + docker { + docker.fixOwnership = true + docker.enabled = true + // docker.userEmulation = true + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + + local { + // This config is for local processing. + process { + maxMemory = 25.GB + withLabel: verylowcpu { cpus = 2 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 6 } + withLabel: highcpu { cpus = 12 } + + withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 12.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 20.GB * task.attempt ) } } + } + } +} + +def get_memory(to_compare) { + if (!process.containsKey("maxMemory") || !process.maxMemory) { + return to_compare + } + + try { + if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { + return process.maxMemory + } + else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { + return max_memory as nextflow.util.MemoryUnit + } + else { + return to_compare + } + } catch (all) { + println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" + System.exit(1) + } + } diff --git a/target/nextflow/demultiplex/.config.vsh.yaml b/target/nextflow/demultiplex/.config.vsh.yaml index ca29045..5a2a2e2 100644 --- a/target/nextflow/demultiplex/.config.vsh.yaml +++ b/target/nextflow/demultiplex/.config.vsh.yaml @@ -1,5 +1,5 @@ name: "demultiplex" -version: "v0.3.3" +version: "v0.3.4" argument_groups: - name: "Input arguments" arguments: @@ -101,6 +101,9 @@ resources: path: "main.nf" is_executable: true entrypoint: "run_wf" +- type: "file" + path: "nextflow_labels.config" + dest: "nextflow_labels.config" description: "Demultiplexing of raw sequencing data" test_resources: - type: "nextflow_script" @@ -217,6 +220,8 @@ runners: cpu200: "cpus = 200" cpu500: "cpus = 500" cpu1000: "cpus = 1000" + script: + - "includeConfig(\"nextflow_labels.config\")" debug: false container: "docker" engines: @@ -231,9 +236,9 @@ build_info: output: "target/nextflow/demultiplex" executable: "target/nextflow/demultiplex/main.nf" viash_version: "0.9.0" - git_commit: "e297e8ff45464065384476fe521ab2700ec3739c" - git_remote: "https://x-access-token:ghs_44flLTQIYGBkc9xcwCZoLYe7mxXzgw1iszFO@github.com/viash-hub/demultiplex" - git_tag: "v0.3.2-4-ge297e8f" + git_commit: "e6f8b20d70be8f47907c08cc3c7802920cc0eee4" + git_remote: "https://x-access-token:ghs_FhW4M4TmdkVUw8Om3VZ1e7pO685xf7028pZ3@github.com/viash-hub/demultiplex" + git_tag: "v0.3.3-3-ge6f8b20" dependencies: - "target/nextflow/io/untar" - "target/nextflow/dataflow/gather_fastqs_and_validate" @@ -245,7 +250,7 @@ build_info: - "target/dependencies/vsh/vsh/biobox/v0.3.0/nextflow/multiqc" package_config: name: "demultiplex" - version: "v0.3.3" + version: "v0.3.4" description: "Demultiplexing pipeline\n" info: test_resources: @@ -256,10 +261,12 @@ package_config: target: "target" config_mods: - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\ - \ := '$id'\n" + \ := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n\ + .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ + )'\n" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.3.3'" + - ".engines[.type == 'docker'].target_tag := 'v0.3.4'" keywords: - "bioinformatics" - "sequence" diff --git a/target/nextflow/demultiplex/main.nf b/target/nextflow/demultiplex/main.nf index b4b94d6..b713c1d 100644 --- a/target/nextflow/demultiplex/main.nf +++ b/target/nextflow/demultiplex/main.nf @@ -1,4 +1,4 @@ -// demultiplex v0.3.3 +// demultiplex v0.3.4 // // This wrapper script is auto-generated by viash 0.9.0 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -2805,7 +2805,7 @@ meta = [ "resources_dir": moduleDir.toRealPath().normalize(), "config": processConfig(readJsonBlob('''{ "name" : "demultiplex", - "version" : "v0.3.3", + "version" : "v0.3.4", "argument_groups" : [ { "name" : "Input arguments", @@ -2923,6 +2923,11 @@ meta = [ "path" : "main.nf", "is_executable" : true, "entrypoint" : "run_wf" + }, + { + "type" : "file", + "path" : "/src/config/labels.config", + "dest" : "nextflow_labels.config" } ], "description" : "Demultiplexing of raw sequencing data", @@ -3079,7 +3084,10 @@ meta = [ "cpu200" : "cpus = 200", "cpu500" : "cpus = 500", "cpu1000" : "cpus = 1000" - } + }, + "script" : [ + "includeConfig(\\"nextflow_labels.config\\")" + ] }, "debug" : false, "container" : "docker" @@ -3101,13 +3109,13 @@ meta = [ "engine" : "native|native", "output" : "target/nextflow/demultiplex", "viash_version" : "0.9.0", - "git_commit" : "e297e8ff45464065384476fe521ab2700ec3739c", - "git_remote" : "https://x-access-token:ghs_44flLTQIYGBkc9xcwCZoLYe7mxXzgw1iszFO@github.com/viash-hub/demultiplex", - "git_tag" : "v0.3.2-4-ge297e8f" + "git_commit" : "e6f8b20d70be8f47907c08cc3c7802920cc0eee4", + "git_remote" : "https://x-access-token:ghs_FhW4M4TmdkVUw8Om3VZ1e7pO685xf7028pZ3@github.com/viash-hub/demultiplex", + "git_tag" : "v0.3.3-3-ge6f8b20" }, "package_config" : { "name" : "demultiplex", - "version" : "v0.3.3", + "version" : "v0.3.4", "description" : "Demultiplexing pipeline\n", "info" : { "test_resources" : [ @@ -3121,10 +3129,10 @@ meta = [ "source" : "src", "target" : "target", "config_mods" : [ - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n", + ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v0.3.3'" + ".engines[.type == 'docker'].target_tag := 'v0.3.4'" ], "keywords" : [ "bioinformatics", diff --git a/target/nextflow/demultiplex/nextflow.config b/target/nextflow/demultiplex/nextflow.config index 51af71b..b7a1762 100644 --- a/target/nextflow/demultiplex/nextflow.config +++ b/target/nextflow/demultiplex/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'demultiplex' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v0.3.3' + version = 'v0.3.4' description = 'Demultiplexing of raw sequencing data' } @@ -122,4 +122,4 @@ process{ withLabel: cpu1000 { cpus = 1000 } } - +includeConfig("nextflow_labels.config") diff --git a/target/nextflow/demultiplex/nextflow_labels.config b/target/nextflow/demultiplex/nextflow_labels.config new file mode 100644 index 0000000..e7ab7b8 --- /dev/null +++ b/target/nextflow/demultiplex/nextflow_labels.config @@ -0,0 +1,98 @@ +process { + container = 'nextflow/bash:latest' + + // default resources + memory = { 8.Gb * task.attempt } + cpus = 8 + maxForks = 36 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + maxMemory = 192.GB + + // Resource labels + withLabel: verylowcpu { cpus = 2 } + withLabel: lowcpu { cpus = 8 } + withLabel: midcpu { cpus = 16 } + withLabel: highcpu { cpus = 32 } + + withLabel: verylowmem { memory = { get_memory( 4.GB * task.attempt ) } } + withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 16.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 64.GB * task.attempt ) } } + +} + +profiles { + // detect tempdir + tempDir = java.nio.file.Paths.get( + System.getenv('NXF_TEMP') ?: + System.getenv('VIASH_TEMP') ?: + System.getenv('TEMPDIR') ?: + System.getenv('TMPDIR') ?: + '/tmp' + ).toAbsolutePath() + + mount_temp { + docker.temp = tempDir + podman.temp = tempDir + charliecloud.temp = tempDir + } + + no_publish { + process { + withName: '.*' { + publishDir = [ + enabled: false + ] + } + } + } + + docker { + docker.fixOwnership = true + docker.enabled = true + // docker.userEmulation = true + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + + local { + // This config is for local processing. + process { + maxMemory = 25.GB + withLabel: verylowcpu { cpus = 2 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 6 } + withLabel: highcpu { cpus = 12 } + + withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 12.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 20.GB * task.attempt ) } } + } + } +} + +def get_memory(to_compare) { + if (!process.containsKey("maxMemory") || !process.maxMemory) { + return to_compare + } + + try { + if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { + return process.maxMemory + } + else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { + return max_memory as nextflow.util.MemoryUnit + } + else { + return to_compare + } + } catch (all) { + println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" + System.exit(1) + } + } diff --git a/target/nextflow/io/interop_summary_to_csv/.config.vsh.yaml b/target/nextflow/io/interop_summary_to_csv/.config.vsh.yaml index f589f95..3565d57 100644 --- a/target/nextflow/io/interop_summary_to_csv/.config.vsh.yaml +++ b/target/nextflow/io/interop_summary_to_csv/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "interop_summary_to_csv" namespace: "io" -version: "v0.3.3" +version: "v0.3.4" argument_groups: - name: "Input arguments" arguments: @@ -38,6 +38,9 @@ resources: - type: "bash_script" path: "script.sh" is_executable: true +- type: "file" + path: "nextflow_labels.config" + dest: "nextflow_labels.config" info: null status: "enabled" requirements: @@ -109,6 +112,8 @@ runners: cpu200: "cpus = 200" cpu500: "cpus = 500" cpu1000: "cpus = 1000" + script: + - "includeConfig(\"nextflow_labels.config\")" debug: false container: "docker" engines: @@ -116,7 +121,7 @@ engines: id: "docker" image: "debian:stable-slim" target_registry: "images.viash-hub.com" - target_tag: "v0.3.3" + target_tag: "v0.3.4" namespace_separator: "/" setup: - type: "apt" @@ -141,12 +146,12 @@ build_info: output: "target/nextflow/io/interop_summary_to_csv" executable: "target/nextflow/io/interop_summary_to_csv/main.nf" viash_version: "0.9.0" - git_commit: "e297e8ff45464065384476fe521ab2700ec3739c" - git_remote: "https://x-access-token:ghs_44flLTQIYGBkc9xcwCZoLYe7mxXzgw1iszFO@github.com/viash-hub/demultiplex" - git_tag: "v0.3.2-4-ge297e8f" + git_commit: "e6f8b20d70be8f47907c08cc3c7802920cc0eee4" + git_remote: "https://x-access-token:ghs_FhW4M4TmdkVUw8Om3VZ1e7pO685xf7028pZ3@github.com/viash-hub/demultiplex" + git_tag: "v0.3.3-3-ge6f8b20" package_config: name: "demultiplex" - version: "v0.3.3" + version: "v0.3.4" description: "Demultiplexing pipeline\n" info: test_resources: @@ -157,10 +162,12 @@ package_config: target: "target" config_mods: - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\ - \ := '$id'\n" + \ := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n\ + .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ + )'\n" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.3.3'" + - ".engines[.type == 'docker'].target_tag := 'v0.3.4'" keywords: - "bioinformatics" - "sequence" diff --git a/target/nextflow/io/interop_summary_to_csv/main.nf b/target/nextflow/io/interop_summary_to_csv/main.nf index a018462..f9a9b5a 100644 --- a/target/nextflow/io/interop_summary_to_csv/main.nf +++ b/target/nextflow/io/interop_summary_to_csv/main.nf @@ -1,4 +1,4 @@ -// interop_summary_to_csv v0.3.3 +// interop_summary_to_csv v0.3.4 // // This wrapper script is auto-generated by viash 0.9.0 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -2806,7 +2806,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "interop_summary_to_csv", "namespace" : "io", - "version" : "v0.3.3", + "version" : "v0.3.4", "argument_groups" : [ { "name" : "Input arguments", @@ -2855,6 +2855,11 @@ meta = [ "type" : "bash_script", "path" : "script.sh", "is_executable" : true + }, + { + "type" : "file", + "path" : "/src/config/labels.config", + "dest" : "nextflow_labels.config" } ], "status" : "enabled", @@ -2935,7 +2940,10 @@ meta = [ "cpu200" : "cpus = 200", "cpu500" : "cpus = 500", "cpu1000" : "cpus = 1000" - } + }, + "script" : [ + "includeConfig(\\"nextflow_labels.config\\")" + ] }, "debug" : false, "container" : "docker" @@ -2947,7 +2955,7 @@ meta = [ "id" : "docker", "image" : "debian:stable-slim", "target_registry" : "images.viash-hub.com", - "target_tag" : "v0.3.3", + "target_tag" : "v0.3.4", "namespace_separator" : "/", "setup" : [ { @@ -2977,13 +2985,13 @@ meta = [ "engine" : "docker|native", "output" : "target/nextflow/io/interop_summary_to_csv", "viash_version" : "0.9.0", - "git_commit" : "e297e8ff45464065384476fe521ab2700ec3739c", - "git_remote" : "https://x-access-token:ghs_44flLTQIYGBkc9xcwCZoLYe7mxXzgw1iszFO@github.com/viash-hub/demultiplex", - "git_tag" : "v0.3.2-4-ge297e8f" + "git_commit" : "e6f8b20d70be8f47907c08cc3c7802920cc0eee4", + "git_remote" : "https://x-access-token:ghs_FhW4M4TmdkVUw8Om3VZ1e7pO685xf7028pZ3@github.com/viash-hub/demultiplex", + "git_tag" : "v0.3.3-3-ge6f8b20" }, "package_config" : { "name" : "demultiplex", - "version" : "v0.3.3", + "version" : "v0.3.4", "description" : "Demultiplexing pipeline\n", "info" : { "test_resources" : [ @@ -2997,10 +3005,10 @@ meta = [ "source" : "src", "target" : "target", "config_mods" : [ - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n", + ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v0.3.3'" + ".engines[.type == 'docker'].target_tag := 'v0.3.4'" ], "keywords" : [ "bioinformatics", @@ -3423,7 +3431,7 @@ meta["defaults"] = [ "container" : { "registry" : "images.viash-hub.com", "image" : "vsh/demultiplex/io/interop_summary_to_csv", - "tag" : "v0.3.3" + "tag" : "v0.3.4" }, "tag" : "$id" }'''), diff --git a/target/nextflow/io/interop_summary_to_csv/nextflow.config b/target/nextflow/io/interop_summary_to_csv/nextflow.config index 8d86c58..f384448 100644 --- a/target/nextflow/io/interop_summary_to_csv/nextflow.config +++ b/target/nextflow/io/interop_summary_to_csv/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'io/interop_summary_to_csv' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v0.3.3' + version = 'v0.3.4' } process.container = 'nextflow/bash:latest' @@ -121,4 +121,4 @@ process{ withLabel: cpu1000 { cpus = 1000 } } - +includeConfig("nextflow_labels.config") diff --git a/target/nextflow/io/interop_summary_to_csv/nextflow_labels.config b/target/nextflow/io/interop_summary_to_csv/nextflow_labels.config new file mode 100644 index 0000000..e7ab7b8 --- /dev/null +++ b/target/nextflow/io/interop_summary_to_csv/nextflow_labels.config @@ -0,0 +1,98 @@ +process { + container = 'nextflow/bash:latest' + + // default resources + memory = { 8.Gb * task.attempt } + cpus = 8 + maxForks = 36 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + maxMemory = 192.GB + + // Resource labels + withLabel: verylowcpu { cpus = 2 } + withLabel: lowcpu { cpus = 8 } + withLabel: midcpu { cpus = 16 } + withLabel: highcpu { cpus = 32 } + + withLabel: verylowmem { memory = { get_memory( 4.GB * task.attempt ) } } + withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 16.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 64.GB * task.attempt ) } } + +} + +profiles { + // detect tempdir + tempDir = java.nio.file.Paths.get( + System.getenv('NXF_TEMP') ?: + System.getenv('VIASH_TEMP') ?: + System.getenv('TEMPDIR') ?: + System.getenv('TMPDIR') ?: + '/tmp' + ).toAbsolutePath() + + mount_temp { + docker.temp = tempDir + podman.temp = tempDir + charliecloud.temp = tempDir + } + + no_publish { + process { + withName: '.*' { + publishDir = [ + enabled: false + ] + } + } + } + + docker { + docker.fixOwnership = true + docker.enabled = true + // docker.userEmulation = true + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + + local { + // This config is for local processing. + process { + maxMemory = 25.GB + withLabel: verylowcpu { cpus = 2 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 6 } + withLabel: highcpu { cpus = 12 } + + withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 12.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 20.GB * task.attempt ) } } + } + } +} + +def get_memory(to_compare) { + if (!process.containsKey("maxMemory") || !process.maxMemory) { + return to_compare + } + + try { + if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { + return process.maxMemory + } + else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { + return max_memory as nextflow.util.MemoryUnit + } + else { + return to_compare + } + } catch (all) { + println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" + System.exit(1) + } + } diff --git a/target/nextflow/io/publish/.config.vsh.yaml b/target/nextflow/io/publish/.config.vsh.yaml index 4cd6e89..88fd47e 100644 --- a/target/nextflow/io/publish/.config.vsh.yaml +++ b/target/nextflow/io/publish/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "publish" namespace: "io" -version: "v0.3.3" +version: "v0.3.4" argument_groups: - name: "Input arguments" arguments: @@ -94,6 +94,9 @@ resources: - type: "bash_script" path: "code.sh" is_executable: true +- type: "file" + path: "nextflow_labels.config" + dest: "nextflow_labels.config" description: "Publish the processed results of the run" info: null status: "enabled" @@ -166,6 +169,8 @@ runners: cpu200: "cpus = 200" cpu500: "cpus = 500" cpu1000: "cpus = 1000" + script: + - "includeConfig(\"nextflow_labels.config\")" debug: false container: "docker" engines: @@ -173,7 +178,7 @@ engines: id: "docker" image: "debian:stable-slim" target_registry: "images.viash-hub.com" - target_tag: "v0.3.3" + target_tag: "v0.3.4" namespace_separator: "/" setup: - type: "apt" @@ -191,12 +196,12 @@ build_info: output: "target/nextflow/io/publish" executable: "target/nextflow/io/publish/main.nf" viash_version: "0.9.0" - git_commit: "e297e8ff45464065384476fe521ab2700ec3739c" - git_remote: "https://x-access-token:ghs_44flLTQIYGBkc9xcwCZoLYe7mxXzgw1iszFO@github.com/viash-hub/demultiplex" - git_tag: "v0.3.2-4-ge297e8f" + git_commit: "e6f8b20d70be8f47907c08cc3c7802920cc0eee4" + git_remote: "https://x-access-token:ghs_FhW4M4TmdkVUw8Om3VZ1e7pO685xf7028pZ3@github.com/viash-hub/demultiplex" + git_tag: "v0.3.3-3-ge6f8b20" package_config: name: "demultiplex" - version: "v0.3.3" + version: "v0.3.4" description: "Demultiplexing pipeline\n" info: test_resources: @@ -207,10 +212,12 @@ package_config: target: "target" config_mods: - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\ - \ := '$id'\n" + \ := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n\ + .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ + )'\n" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.3.3'" + - ".engines[.type == 'docker'].target_tag := 'v0.3.4'" keywords: - "bioinformatics" - "sequence" diff --git a/target/nextflow/io/publish/main.nf b/target/nextflow/io/publish/main.nf index 8d089e5..4a1ad03 100644 --- a/target/nextflow/io/publish/main.nf +++ b/target/nextflow/io/publish/main.nf @@ -1,4 +1,4 @@ -// publish v0.3.3 +// publish v0.3.4 // // This wrapper script is auto-generated by viash 0.9.0 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -2806,7 +2806,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "publish", "namespace" : "io", - "version" : "v0.3.3", + "version" : "v0.3.4", "argument_groups" : [ { "name" : "Input arguments", @@ -2920,6 +2920,11 @@ meta = [ "type" : "bash_script", "path" : "./code.sh", "is_executable" : true + }, + { + "type" : "file", + "path" : "/src/config/labels.config", + "dest" : "nextflow_labels.config" } ], "description" : "Publish the processed results of the run", @@ -3001,7 +3006,10 @@ meta = [ "cpu200" : "cpus = 200", "cpu500" : "cpus = 500", "cpu1000" : "cpus = 1000" - } + }, + "script" : [ + "includeConfig(\\"nextflow_labels.config\\")" + ] }, "debug" : false, "container" : "docker" @@ -3013,7 +3021,7 @@ meta = [ "id" : "docker", "image" : "debian:stable-slim", "target_registry" : "images.viash-hub.com", - "target_tag" : "v0.3.3", + "target_tag" : "v0.3.4", "namespace_separator" : "/", "setup" : [ { @@ -3036,13 +3044,13 @@ meta = [ "engine" : "docker|native", "output" : "target/nextflow/io/publish", "viash_version" : "0.9.0", - "git_commit" : "e297e8ff45464065384476fe521ab2700ec3739c", - "git_remote" : "https://x-access-token:ghs_44flLTQIYGBkc9xcwCZoLYe7mxXzgw1iszFO@github.com/viash-hub/demultiplex", - "git_tag" : "v0.3.2-4-ge297e8f" + "git_commit" : "e6f8b20d70be8f47907c08cc3c7802920cc0eee4", + "git_remote" : "https://x-access-token:ghs_FhW4M4TmdkVUw8Om3VZ1e7pO685xf7028pZ3@github.com/viash-hub/demultiplex", + "git_tag" : "v0.3.3-3-ge6f8b20" }, "package_config" : { "name" : "demultiplex", - "version" : "v0.3.3", + "version" : "v0.3.4", "description" : "Demultiplexing pipeline\n", "info" : { "test_resources" : [ @@ -3056,10 +3064,10 @@ meta = [ "source" : "src", "target" : "target", "config_mods" : [ - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n", + ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v0.3.3'" + ".engines[.type == 'docker'].target_tag := 'v0.3.4'" ], "keywords" : [ "bioinformatics", @@ -3503,7 +3511,7 @@ meta["defaults"] = [ "container" : { "registry" : "images.viash-hub.com", "image" : "vsh/demultiplex/io/publish", - "tag" : "v0.3.3" + "tag" : "v0.3.4" }, "tag" : "$id" }'''), diff --git a/target/nextflow/io/publish/nextflow.config b/target/nextflow/io/publish/nextflow.config index c96ca53..0cfd662 100644 --- a/target/nextflow/io/publish/nextflow.config +++ b/target/nextflow/io/publish/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'io/publish' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v0.3.3' + version = 'v0.3.4' description = 'Publish the processed results of the run' } @@ -122,4 +122,4 @@ process{ withLabel: cpu1000 { cpus = 1000 } } - +includeConfig("nextflow_labels.config") diff --git a/target/nextflow/io/publish/nextflow_labels.config b/target/nextflow/io/publish/nextflow_labels.config new file mode 100644 index 0000000..e7ab7b8 --- /dev/null +++ b/target/nextflow/io/publish/nextflow_labels.config @@ -0,0 +1,98 @@ +process { + container = 'nextflow/bash:latest' + + // default resources + memory = { 8.Gb * task.attempt } + cpus = 8 + maxForks = 36 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + maxMemory = 192.GB + + // Resource labels + withLabel: verylowcpu { cpus = 2 } + withLabel: lowcpu { cpus = 8 } + withLabel: midcpu { cpus = 16 } + withLabel: highcpu { cpus = 32 } + + withLabel: verylowmem { memory = { get_memory( 4.GB * task.attempt ) } } + withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 16.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 64.GB * task.attempt ) } } + +} + +profiles { + // detect tempdir + tempDir = java.nio.file.Paths.get( + System.getenv('NXF_TEMP') ?: + System.getenv('VIASH_TEMP') ?: + System.getenv('TEMPDIR') ?: + System.getenv('TMPDIR') ?: + '/tmp' + ).toAbsolutePath() + + mount_temp { + docker.temp = tempDir + podman.temp = tempDir + charliecloud.temp = tempDir + } + + no_publish { + process { + withName: '.*' { + publishDir = [ + enabled: false + ] + } + } + } + + docker { + docker.fixOwnership = true + docker.enabled = true + // docker.userEmulation = true + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + + local { + // This config is for local processing. + process { + maxMemory = 25.GB + withLabel: verylowcpu { cpus = 2 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 6 } + withLabel: highcpu { cpus = 12 } + + withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 12.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 20.GB * task.attempt ) } } + } + } +} + +def get_memory(to_compare) { + if (!process.containsKey("maxMemory") || !process.maxMemory) { + return to_compare + } + + try { + if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { + return process.maxMemory + } + else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { + return max_memory as nextflow.util.MemoryUnit + } + else { + return to_compare + } + } catch (all) { + println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" + System.exit(1) + } + } diff --git a/target/nextflow/io/untar/.config.vsh.yaml b/target/nextflow/io/untar/.config.vsh.yaml index 5a0c730..1c5560b 100644 --- a/target/nextflow/io/untar/.config.vsh.yaml +++ b/target/nextflow/io/untar/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "untar" namespace: "io" -version: "v0.3.3" +version: "v0.3.4" argument_groups: - name: "Input arguments" arguments: @@ -45,6 +45,9 @@ resources: - type: "bash_script" path: "script.sh" is_executable: true +- type: "file" + path: "nextflow_labels.config" + dest: "nextflow_labels.config" description: "Unpack a .tar file. When the contents of the .tar file is just a single\ \ directory,\nput the contents of the directory into the output folder instead of\ \ that directory.\n" @@ -123,6 +126,8 @@ runners: cpu200: "cpus = 200" cpu500: "cpus = 500" cpu1000: "cpus = 1000" + script: + - "includeConfig(\"nextflow_labels.config\")" debug: false container: "docker" engines: @@ -130,7 +135,7 @@ engines: id: "docker" image: "debian:stable-slim" target_registry: "images.viash-hub.com" - target_tag: "v0.3.3" + target_tag: "v0.3.4" namespace_separator: "/" setup: - type: "apt" @@ -148,12 +153,12 @@ build_info: output: "target/nextflow/io/untar" executable: "target/nextflow/io/untar/main.nf" viash_version: "0.9.0" - git_commit: "e297e8ff45464065384476fe521ab2700ec3739c" - git_remote: "https://x-access-token:ghs_44flLTQIYGBkc9xcwCZoLYe7mxXzgw1iszFO@github.com/viash-hub/demultiplex" - git_tag: "v0.3.2-4-ge297e8f" + git_commit: "e6f8b20d70be8f47907c08cc3c7802920cc0eee4" + git_remote: "https://x-access-token:ghs_FhW4M4TmdkVUw8Om3VZ1e7pO685xf7028pZ3@github.com/viash-hub/demultiplex" + git_tag: "v0.3.3-3-ge6f8b20" package_config: name: "demultiplex" - version: "v0.3.3" + version: "v0.3.4" description: "Demultiplexing pipeline\n" info: test_resources: @@ -164,10 +169,12 @@ package_config: target: "target" config_mods: - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\ - \ := '$id'\n" + \ := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n\ + .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ + )'\n" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.3.3'" + - ".engines[.type == 'docker'].target_tag := 'v0.3.4'" keywords: - "bioinformatics" - "sequence" diff --git a/target/nextflow/io/untar/main.nf b/target/nextflow/io/untar/main.nf index 06664d1..4f0ae1d 100644 --- a/target/nextflow/io/untar/main.nf +++ b/target/nextflow/io/untar/main.nf @@ -1,4 +1,4 @@ -// untar v0.3.3 +// untar v0.3.4 // // This wrapper script is auto-generated by viash 0.9.0 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -2806,7 +2806,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "untar", "namespace" : "io", - "version" : "v0.3.3", + "version" : "v0.3.4", "argument_groups" : [ { "name" : "Input arguments", @@ -2866,6 +2866,11 @@ meta = [ "type" : "bash_script", "path" : "script.sh", "is_executable" : true + }, + { + "type" : "file", + "path" : "/src/config/labels.config", + "dest" : "nextflow_labels.config" } ], "description" : "Unpack a .tar file. When the contents of the .tar file is just a single directory,\nput the contents of the directory into the output folder instead of that directory.\n", @@ -2954,7 +2959,10 @@ meta = [ "cpu200" : "cpus = 200", "cpu500" : "cpus = 500", "cpu1000" : "cpus = 1000" - } + }, + "script" : [ + "includeConfig(\\"nextflow_labels.config\\")" + ] }, "debug" : false, "container" : "docker" @@ -2966,7 +2974,7 @@ meta = [ "id" : "docker", "image" : "debian:stable-slim", "target_registry" : "images.viash-hub.com", - "target_tag" : "v0.3.3", + "target_tag" : "v0.3.4", "namespace_separator" : "/", "setup" : [ { @@ -2989,13 +2997,13 @@ meta = [ "engine" : "docker|native", "output" : "target/nextflow/io/untar", "viash_version" : "0.9.0", - "git_commit" : "e297e8ff45464065384476fe521ab2700ec3739c", - "git_remote" : "https://x-access-token:ghs_44flLTQIYGBkc9xcwCZoLYe7mxXzgw1iszFO@github.com/viash-hub/demultiplex", - "git_tag" : "v0.3.2-4-ge297e8f" + "git_commit" : "e6f8b20d70be8f47907c08cc3c7802920cc0eee4", + "git_remote" : "https://x-access-token:ghs_FhW4M4TmdkVUw8Om3VZ1e7pO685xf7028pZ3@github.com/viash-hub/demultiplex", + "git_tag" : "v0.3.3-3-ge6f8b20" }, "package_config" : { "name" : "demultiplex", - "version" : "v0.3.3", + "version" : "v0.3.4", "description" : "Demultiplexing pipeline\n", "info" : { "test_resources" : [ @@ -3009,10 +3017,10 @@ meta = [ "source" : "src", "target" : "target", "config_mods" : [ - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n", + ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v0.3.3'" + ".engines[.type == 'docker'].target_tag := 'v0.3.4'" ], "keywords" : [ "bioinformatics", @@ -3465,7 +3473,7 @@ meta["defaults"] = [ "container" : { "registry" : "images.viash-hub.com", "image" : "vsh/demultiplex/io/untar", - "tag" : "v0.3.3" + "tag" : "v0.3.4" }, "tag" : "$id" }'''), diff --git a/target/nextflow/io/untar/nextflow.config b/target/nextflow/io/untar/nextflow.config index 44f0c79..564d4b7 100644 --- a/target/nextflow/io/untar/nextflow.config +++ b/target/nextflow/io/untar/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'io/untar' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v0.3.3' + version = 'v0.3.4' description = 'Unpack a .tar file. When the contents of the .tar file is just a single directory,\nput the contents of the directory into the output folder instead of that directory.\n' } @@ -122,4 +122,4 @@ process{ withLabel: cpu1000 { cpus = 1000 } } - +includeConfig("nextflow_labels.config") diff --git a/target/nextflow/io/untar/nextflow_labels.config b/target/nextflow/io/untar/nextflow_labels.config new file mode 100644 index 0000000..e7ab7b8 --- /dev/null +++ b/target/nextflow/io/untar/nextflow_labels.config @@ -0,0 +1,98 @@ +process { + container = 'nextflow/bash:latest' + + // default resources + memory = { 8.Gb * task.attempt } + cpus = 8 + maxForks = 36 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + maxMemory = 192.GB + + // Resource labels + withLabel: verylowcpu { cpus = 2 } + withLabel: lowcpu { cpus = 8 } + withLabel: midcpu { cpus = 16 } + withLabel: highcpu { cpus = 32 } + + withLabel: verylowmem { memory = { get_memory( 4.GB * task.attempt ) } } + withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 16.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 64.GB * task.attempt ) } } + +} + +profiles { + // detect tempdir + tempDir = java.nio.file.Paths.get( + System.getenv('NXF_TEMP') ?: + System.getenv('VIASH_TEMP') ?: + System.getenv('TEMPDIR') ?: + System.getenv('TMPDIR') ?: + '/tmp' + ).toAbsolutePath() + + mount_temp { + docker.temp = tempDir + podman.temp = tempDir + charliecloud.temp = tempDir + } + + no_publish { + process { + withName: '.*' { + publishDir = [ + enabled: false + ] + } + } + } + + docker { + docker.fixOwnership = true + docker.enabled = true + // docker.userEmulation = true + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + + local { + // This config is for local processing. + process { + maxMemory = 25.GB + withLabel: verylowcpu { cpus = 2 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 6 } + withLabel: highcpu { cpus = 12 } + + withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 12.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 20.GB * task.attempt ) } } + } + } +} + +def get_memory(to_compare) { + if (!process.containsKey("maxMemory") || !process.maxMemory) { + return to_compare + } + + try { + if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { + return process.maxMemory + } + else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { + return max_memory as nextflow.util.MemoryUnit + } + else { + return to_compare + } + } catch (all) { + println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" + System.exit(1) + } + } diff --git a/target/nextflow/runner/.config.vsh.yaml b/target/nextflow/runner/.config.vsh.yaml index 4d79241..1840269 100644 --- a/target/nextflow/runner/.config.vsh.yaml +++ b/target/nextflow/runner/.config.vsh.yaml @@ -1,5 +1,5 @@ name: "runner" -version: "v0.3.3" +version: "v0.3.4" argument_groups: - name: "Input arguments" arguments: @@ -89,6 +89,9 @@ resources: path: "main.nf" is_executable: true entrypoint: "run_wf" +- type: "file" + path: "nextflow_labels.config" + dest: "nextflow_labels.config" description: "Runner for demultiplexing of raw sequencing data" info: null status: "enabled" @@ -165,6 +168,8 @@ runners: cpu200: "cpus = 200" cpu500: "cpus = 500" cpu1000: "cpus = 1000" + script: + - "includeConfig(\"nextflow_labels.config\")" debug: false container: "docker" engines: @@ -179,15 +184,15 @@ build_info: output: "target/nextflow/runner" executable: "target/nextflow/runner/main.nf" viash_version: "0.9.0" - git_commit: "e297e8ff45464065384476fe521ab2700ec3739c" - git_remote: "https://x-access-token:ghs_44flLTQIYGBkc9xcwCZoLYe7mxXzgw1iszFO@github.com/viash-hub/demultiplex" - git_tag: "v0.3.2-4-ge297e8f" + git_commit: "e6f8b20d70be8f47907c08cc3c7802920cc0eee4" + git_remote: "https://x-access-token:ghs_FhW4M4TmdkVUw8Om3VZ1e7pO685xf7028pZ3@github.com/viash-hub/demultiplex" + git_tag: "v0.3.3-3-ge6f8b20" dependencies: - "target/nextflow/demultiplex" - "target/nextflow/io/publish" package_config: name: "demultiplex" - version: "v0.3.3" + version: "v0.3.4" description: "Demultiplexing pipeline\n" info: test_resources: @@ -198,10 +203,12 @@ package_config: target: "target" config_mods: - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\ - \ := '$id'\n" + \ := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n\ + .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ + )'\n" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.3.3'" + - ".engines[.type == 'docker'].target_tag := 'v0.3.4'" keywords: - "bioinformatics" - "sequence" diff --git a/target/nextflow/runner/main.nf b/target/nextflow/runner/main.nf index d06ecec..b4cc445 100644 --- a/target/nextflow/runner/main.nf +++ b/target/nextflow/runner/main.nf @@ -1,4 +1,4 @@ -// runner v0.3.3 +// runner v0.3.4 // // This wrapper script is auto-generated by viash 0.9.0 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -2805,7 +2805,7 @@ meta = [ "resources_dir": moduleDir.toRealPath().normalize(), "config": processConfig(readJsonBlob('''{ "name" : "runner", - "version" : "v0.3.3", + "version" : "v0.3.4", "argument_groups" : [ { "name" : "Input arguments", @@ -2909,6 +2909,11 @@ meta = [ "path" : "main.nf", "is_executable" : true, "entrypoint" : "run_wf" + }, + { + "type" : "file", + "path" : "/src/config/labels.config", + "dest" : "nextflow_labels.config" } ], "description" : "Runner for demultiplexing of raw sequencing data", @@ -2999,7 +3004,10 @@ meta = [ "cpu200" : "cpus = 200", "cpu500" : "cpus = 500", "cpu1000" : "cpus = 1000" - } + }, + "script" : [ + "includeConfig(\\"nextflow_labels.config\\")" + ] }, "debug" : false, "container" : "docker" @@ -3021,13 +3029,13 @@ meta = [ "engine" : "native|native", "output" : "target/nextflow/runner", "viash_version" : "0.9.0", - "git_commit" : "e297e8ff45464065384476fe521ab2700ec3739c", - "git_remote" : "https://x-access-token:ghs_44flLTQIYGBkc9xcwCZoLYe7mxXzgw1iszFO@github.com/viash-hub/demultiplex", - "git_tag" : "v0.3.2-4-ge297e8f" + "git_commit" : "e6f8b20d70be8f47907c08cc3c7802920cc0eee4", + "git_remote" : "https://x-access-token:ghs_FhW4M4TmdkVUw8Om3VZ1e7pO685xf7028pZ3@github.com/viash-hub/demultiplex", + "git_tag" : "v0.3.3-3-ge6f8b20" }, "package_config" : { "name" : "demultiplex", - "version" : "v0.3.3", + "version" : "v0.3.4", "description" : "Demultiplexing pipeline\n", "info" : { "test_resources" : [ @@ -3041,10 +3049,10 @@ meta = [ "source" : "src", "target" : "target", "config_mods" : [ - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n", + ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v0.3.3'" + ".engines[.type == 'docker'].target_tag := 'v0.3.4'" ], "keywords" : [ "bioinformatics", diff --git a/target/nextflow/runner/nextflow.config b/target/nextflow/runner/nextflow.config index 1fc1d9f..153f65f 100644 --- a/target/nextflow/runner/nextflow.config +++ b/target/nextflow/runner/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'runner' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v0.3.3' + version = 'v0.3.4' description = 'Runner for demultiplexing of raw sequencing data' } @@ -122,4 +122,4 @@ process{ withLabel: cpu1000 { cpus = 1000 } } - +includeConfig("nextflow_labels.config") diff --git a/target/nextflow/runner/nextflow_labels.config b/target/nextflow/runner/nextflow_labels.config new file mode 100644 index 0000000..e7ab7b8 --- /dev/null +++ b/target/nextflow/runner/nextflow_labels.config @@ -0,0 +1,98 @@ +process { + container = 'nextflow/bash:latest' + + // default resources + memory = { 8.Gb * task.attempt } + cpus = 8 + maxForks = 36 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + maxMemory = 192.GB + + // Resource labels + withLabel: verylowcpu { cpus = 2 } + withLabel: lowcpu { cpus = 8 } + withLabel: midcpu { cpus = 16 } + withLabel: highcpu { cpus = 32 } + + withLabel: verylowmem { memory = { get_memory( 4.GB * task.attempt ) } } + withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 16.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 64.GB * task.attempt ) } } + +} + +profiles { + // detect tempdir + tempDir = java.nio.file.Paths.get( + System.getenv('NXF_TEMP') ?: + System.getenv('VIASH_TEMP') ?: + System.getenv('TEMPDIR') ?: + System.getenv('TMPDIR') ?: + '/tmp' + ).toAbsolutePath() + + mount_temp { + docker.temp = tempDir + podman.temp = tempDir + charliecloud.temp = tempDir + } + + no_publish { + process { + withName: '.*' { + publishDir = [ + enabled: false + ] + } + } + } + + docker { + docker.fixOwnership = true + docker.enabled = true + // docker.userEmulation = true + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + + local { + // This config is for local processing. + process { + maxMemory = 25.GB + withLabel: verylowcpu { cpus = 2 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 6 } + withLabel: highcpu { cpus = 12 } + + withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 12.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 20.GB * task.attempt ) } } + } + } +} + +def get_memory(to_compare) { + if (!process.containsKey("maxMemory") || !process.maxMemory) { + return to_compare + } + + try { + if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { + return process.maxMemory + } + else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { + return max_memory as nextflow.util.MemoryUnit + } + else { + return to_compare + } + } catch (all) { + println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" + System.exit(1) + } + }