diff --git a/README.md b/README.md index b4bb54d3..7bb25386 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,98 @@ -# Viash-enabled HT-RNAseq pipeline +# HT-RNAseq - A pipeline for processing high-throughput RNA-seq data ## Introduction __TODO__: Add a description of the pipeline here. ## Test data +As test data, we use [a DRUGseq dataset](https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE176150) from the [NCBI Sequence Read Archive](https://www.ncbi.nlm.nih.gov/sra). + +The original data has been (partly) subsampled to reduce the test runtime. We used [seqtk](https://github.com/lh3/seqtk) for this with a seed of 1, e.g.: + +```bash +seqtk sample -s1 orig/SRR14730302/VH02001614_S8_R1_001.fastq.gz 10000 > 10k/SRR14730302/VH02001614_S8_R1_001.fastq.gz +``` + +The data is available at: `gs://viash-hub-test-data/htrnaseq/v1/`: + +``` +❯ gcstree -f viash-hub-test-data/htrnaseq/v1/ +viash-hub-test-data +└── htrnaseq + └── v1 + ├── [ 48] 2-wells.fasta + ├── [465.3K] GSE176150_metadata.csv + ├── 100k + │ ├── SRR14730301 + │ │ ├── [8.5M] VH02001612_S9_R1_001.fastq + │ │ └── [14.9M] VH02001612_S9_R2_001.fastq + │ └── SRR14730302 + │ ├── [8.5M] VH02001614_S8_R1_001.fastq.gz + │ └── [14.9M] VH02001614_S8_R2_001.fastq.gz + ├── 10k + │ ├── SRR14730301 + │ │ ├── [845.4K] VH02001612_S9_R1_001.fastq + │ │ └── [1.5M] VH02001612_S9_R2_001.fastq + │ └── SRR14730302 + │ ├── [845.3K] VH02001614_S8_R1_001.fastq.gz + │ └── [1.5M] VH02001614_S8_R2_001.fastq.gz + └── orig + ├── [20.4G] SRR14730301 + │ └── [20.4G] SRR14730301 + ├── SRR14730301 + │ ├── [9.1G] VH02001612_S9_R1_001.fastq.gz + │ └── [22.0G] VH02001612_S9_R2_001.fastq.gz + ├── [16.9G] SRR14730302 + │ └── [16.9G] SRR14730302 + ├── SRR14730302 + │ ├── [7.6G] VH02001614_S8_R1_001.fastq.gz + │ └── [18.0G] VH02001614_S8_R2_001.fastq.gz + ├── [18.0G] SRR14730303 + │ └── [18.0G] SRR14730303 + ├── SRR14730303 + │ ├── [8.1G] VH02001618_S7_R1_001.fastq.gz + │ └── [19.2G] VH02001618_S7_R2_001.fastq.gz + ├── [16.5G] SRR14730304 + │ └── [16.5G] SRR14730304 + ├── SRR14730304 + │ ├── [7.5G] VH02001700_S6_R1_001.fastq.gz + │ └── [17.8G] VH02001700_S6_R2_001.fastq.gz + ├── [19.0G] SRR14730305 + │ └── [19.0G] SRR14730305 + ├── SRR14730305 + │ ├── [8.4G] VH02001702_S5_R1_001.fastq.gz + │ └── [20.6G] VH02001702_S5_R2_001.fastq.gz + ├── [14.6G] SRR14730306 + │ └── [14.6G] SRR14730306 + ├── SRR14730306 + │ ├── [6.6G] VH02001704_S4_R1_001.fastq.gz + │ └── [16.0G] VH02001704_S4_R2_001.fastq.gz + ├── [21.5G] SRR14730307 + │ └── [21.5G] SRR14730307 + ├── SRR14730307 + │ ├── [9.6G] VH02001708_S3_R1_001.fastq.gz + │ └── [23.2G] VH02001708_S3_R2_001.fastq.gz + ├── [20.7G] SRR14730308 + │ └── [20.7G] SRR14730308 + ├── SRR14730308 + │ ├── [9.3G] VH02001710_S2_R1_001.fastq.gz + │ └── [22.1G] VH02001710_S2_R2_001.fastq.gz + ├── [15.8G] SRR14730309 + │ └── [15.8G] SRR14730309 + └── SRR14730309 + ├── [7.2G] VH02001712_S1_R1_001.fastq.gz + └── [16.9G] VH02001712_S1_R2_001.fastq.gz + +18 directories, 37 files +``` + + +The `orig` directory contains the original fastq files. The fastq files are available for 10k and 100k subsamples in the `10k` and `100k` directories, respectively. + +The `2-wells.fasta` file contains the barcodes for 2 wells. + +## Test run + The pipeline can be run by creating a `params.yaml` file like this: ```yaml diff --git a/_viash.yaml b/_viash.yaml index cc361827..32c317b8 100644 --- a/_viash.yaml +++ b/_viash.yaml @@ -1,6 +1,6 @@ name: htrnaseq description: | - Demultiplexing pipeline [WIP] + High-throughput pipeline [WIP] license: MIT keywords: [bioinformatics, sequence, high-throughput, mapping, counting, pipeline] links: @@ -11,3 +11,5 @@ viash_version: 0.9.0-RC7 config_mods: | .requirements.commands := ['ps'] + .runners[.type == 'nextflow'].config.script := 'includeConfig("nextflow_labels.config")' + .resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'} diff --git a/target/executable/parallel_map/.config.vsh.yaml b/target/executable/parallel_map/.config.vsh.yaml index 01a40d90..de72f69b 100644 --- a/target/executable/parallel_map/.config.vsh.yaml +++ b/target/executable/parallel_map/.config.vsh.yaml @@ -110,6 +110,9 @@ resources: - type: "bash_script" path: "script.sh" is_executable: true +- type: "file" + path: "nextflow_labels.config" + dest: "nextflow_labels.config" description: "Map wells in batch, using STAR\nSpliced Transcripts Alignment to a Reference\ \ (C) Alexander Dobin\nhttps://github.com/alexdobin/STAR\n" test_resources: @@ -187,6 +190,8 @@ runners: cpu200: "cpus = 200" cpu500: "cpus = 500" cpu1000: "cpus = 1000" + script: + - "includeConfig(\"nextflow_labels.config\")" debug: false container: "docker" engines: @@ -231,18 +236,20 @@ build_info: output: "target/executable/parallel_map" executable: "target/executable/parallel_map/parallel_map" viash_version: "0.9.0-RC6" - git_commit: "bdd7091d5b5982b84103a43fece50c608810e0aa" + git_commit: "c918fed99aca1cacf3b2c02ca13c2f0af5ecf3bf" git_remote: "https://github.com/viash-hub/htrnaseq" package_config: name: "htrnaseq" version: "main_pipeline" - description: "Demultiplexing pipeline [WIP]\n" + description: "High-throughput pipeline [WIP]\n" info: null viash_version: "0.9.0-RC6" source: "src" target: "target" config_mods: - - ".requirements.commands := ['ps']\n" + - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].config.script\ + \ := 'includeConfig(\"nextflow_labels.config\")'\n.resources += {path: '/src/config/labels.config',\ + \ dest: 'nextflow_labels.config'}\n" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - ".engines[.type == 'docker'].target_tag := 'main_pipeline'" diff --git a/target/executable/parallel_map/nextflow_labels.config b/target/executable/parallel_map/nextflow_labels.config new file mode 100644 index 00000000..9ea88b1b --- /dev/null +++ b/target/executable/parallel_map/nextflow_labels.config @@ -0,0 +1,43 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + maxMemory = null + + // Resource labels + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } + withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } + +} + +def get_memory(to_compare) { + if (!process.containsKey("maxMemory") || !process.maxMemory) { + return to_compare + } + + try { + if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { + return process.maxMemory + } + else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { + return max_memory as nextflow.util.MemoryUnit + } + else { + return to_compare + } + } catch (all) { + println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" + System.exit(1) + } + } diff --git a/target/executable/parallel_map/parallel_map b/target/executable/parallel_map/parallel_map index 9c7a1066..0866068a 100755 --- a/target/executable/parallel_map/parallel_map +++ b/target/executable/parallel_map/parallel_map @@ -515,9 +515,9 @@ RUN wget -O $STAR_TARGET $STAR_SOURCE && \ rm $STAR_TARGET && rm -rf /tmp/STAR_$STAR_VERSION LABEL org.opencontainers.image.description="Companion container for running component parallel_map" -LABEL org.opencontainers.image.created="2024-08-20T15:27:42Z" +LABEL org.opencontainers.image.created="2024-08-20T15:45:18Z" LABEL org.opencontainers.image.source="https://github.com/viash-hub/htrnaseq" -LABEL org.opencontainers.image.revision="bdd7091d5b5982b84103a43fece50c608810e0aa" +LABEL org.opencontainers.image.revision="c918fed99aca1cacf3b2c02ca13c2f0af5ecf3bf" LABEL org.opencontainers.image.version="main_pipeline" VIASHDOCKER diff --git a/target/nextflow/parallel_map/.config.vsh.yaml b/target/nextflow/parallel_map/.config.vsh.yaml index 08805c79..c491c8ad 100644 --- a/target/nextflow/parallel_map/.config.vsh.yaml +++ b/target/nextflow/parallel_map/.config.vsh.yaml @@ -110,6 +110,9 @@ resources: - type: "bash_script" path: "script.sh" is_executable: true +- type: "file" + path: "nextflow_labels.config" + dest: "nextflow_labels.config" description: "Map wells in batch, using STAR\nSpliced Transcripts Alignment to a Reference\ \ (C) Alexander Dobin\nhttps://github.com/alexdobin/STAR\n" test_resources: @@ -187,6 +190,8 @@ runners: cpu200: "cpus = 200" cpu500: "cpus = 500" cpu1000: "cpus = 1000" + script: + - "includeConfig(\"nextflow_labels.config\")" debug: false container: "docker" engines: @@ -231,18 +236,20 @@ build_info: output: "target/nextflow/parallel_map" executable: "target/nextflow/parallel_map/main.nf" viash_version: "0.9.0-RC6" - git_commit: "bdd7091d5b5982b84103a43fece50c608810e0aa" + git_commit: "c918fed99aca1cacf3b2c02ca13c2f0af5ecf3bf" git_remote: "https://github.com/viash-hub/htrnaseq" package_config: name: "htrnaseq" version: "main_pipeline" - description: "Demultiplexing pipeline [WIP]\n" + description: "High-throughput pipeline [WIP]\n" info: null viash_version: "0.9.0-RC6" source: "src" target: "target" config_mods: - - ".requirements.commands := ['ps']\n" + - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].config.script\ + \ := 'includeConfig(\"nextflow_labels.config\")'\n.resources += {path: '/src/config/labels.config',\ + \ dest: 'nextflow_labels.config'}\n" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - ".engines[.type == 'docker'].target_tag := 'main_pipeline'" diff --git a/target/nextflow/parallel_map/main.nf b/target/nextflow/parallel_map/main.nf index 03d9b20a..fceebbb1 100644 --- a/target/nextflow/parallel_map/main.nf +++ b/target/nextflow/parallel_map/main.nf @@ -2916,6 +2916,11 @@ meta = [ "type" : "bash_script", "path" : "script.sh", "is_executable" : true + }, + { + "type" : "file", + "path" : "/src/config/labels.config", + "dest" : "nextflow_labels.config" } ], "description" : "Map wells in batch, using STAR\nSpliced Transcripts Alignment to a Reference (C) Alexander Dobin\nhttps://github.com/alexdobin/STAR\n", @@ -3004,7 +3009,10 @@ meta = [ "cpu200" : "cpus = 200", "cpu500" : "cpus = 500", "cpu1000" : "cpus = 1000" - } + }, + "script" : [ + "includeConfig(\\"nextflow_labels.config\\")" + ] }, "debug" : false, "container" : "docker" @@ -3059,18 +3067,18 @@ meta = [ "engine" : "docker|native", "output" : "target/nextflow/parallel_map", "viash_version" : "0.9.0-RC6", - "git_commit" : "bdd7091d5b5982b84103a43fece50c608810e0aa", + "git_commit" : "c918fed99aca1cacf3b2c02ca13c2f0af5ecf3bf", "git_remote" : "https://github.com/viash-hub/htrnaseq" }, "package_config" : { "name" : "htrnaseq", "version" : "main_pipeline", - "description" : "Demultiplexing pipeline [WIP]\n", + "description" : "High-throughput pipeline [WIP]\n", "viash_version" : "0.9.0-RC6", "source" : "src", "target" : "target", "config_mods" : [ - ".requirements.commands := ['ps']\n", + ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", ".engines[.type == 'docker'].target_tag := 'main_pipeline'" diff --git a/target/nextflow/parallel_map/nextflow.config b/target/nextflow/parallel_map/nextflow.config index 4544cc14..a78543a3 100644 --- a/target/nextflow/parallel_map/nextflow.config +++ b/target/nextflow/parallel_map/nextflow.config @@ -122,4 +122,4 @@ process{ withLabel: cpu1000 { cpus = 1000 } } - +includeConfig("nextflow_labels.config") diff --git a/target/nextflow/parallel_map/nextflow_labels.config b/target/nextflow/parallel_map/nextflow_labels.config new file mode 100644 index 00000000..9ea88b1b --- /dev/null +++ b/target/nextflow/parallel_map/nextflow_labels.config @@ -0,0 +1,43 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + maxMemory = null + + // Resource labels + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } + withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } + +} + +def get_memory(to_compare) { + if (!process.containsKey("maxMemory") || !process.maxMemory) { + return to_compare + } + + try { + if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { + return process.maxMemory + } + else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { + return max_memory as nextflow.util.MemoryUnit + } + else { + return to_compare + } + } catch (all) { + println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" + System.exit(1) + } + } diff --git a/target/nextflow/workflows/htrnaseq/.config.vsh.yaml b/target/nextflow/workflows/htrnaseq/.config.vsh.yaml index 0397fe7e..9227e803 100644 --- a/target/nextflow/workflows/htrnaseq/.config.vsh.yaml +++ b/target/nextflow/workflows/htrnaseq/.config.vsh.yaml @@ -73,6 +73,9 @@ resources: path: "main.nf" is_executable: true entrypoint: "run_wf" +- type: "file" + path: "nextflow_labels.config" + dest: "nextflow_labels.config" info: null status: "enabled" requirements: @@ -176,6 +179,8 @@ runners: cpu200: "cpus = 200" cpu500: "cpus = 500" cpu1000: "cpus = 1000" + script: + - "includeConfig(\"nextflow_labels.config\")" debug: false container: "docker" engines: @@ -190,7 +195,7 @@ build_info: output: "target/nextflow/workflows/htrnaseq" executable: "target/nextflow/workflows/htrnaseq/main.nf" viash_version: "0.9.0-RC6" - git_commit: "bdd7091d5b5982b84103a43fece50c608810e0aa" + git_commit: "c918fed99aca1cacf3b2c02ca13c2f0af5ecf3bf" git_remote: "https://github.com/viash-hub/htrnaseq" dependencies: - "target/nextflow/workflows/well_demultiplex" @@ -203,13 +208,15 @@ build_info: package_config: name: "htrnaseq" version: "main_pipeline" - description: "Demultiplexing pipeline [WIP]\n" + description: "High-throughput pipeline [WIP]\n" info: null viash_version: "0.9.0-RC6" source: "src" target: "target" config_mods: - - ".requirements.commands := ['ps']\n" + - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].config.script\ + \ := 'includeConfig(\"nextflow_labels.config\")'\n.resources += {path: '/src/config/labels.config',\ + \ dest: 'nextflow_labels.config'}\n" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - ".engines[.type == 'docker'].target_tag := 'main_pipeline'" diff --git a/target/nextflow/workflows/htrnaseq/main.nf b/target/nextflow/workflows/htrnaseq/main.nf index cbdb3cf3..01fc3252 100644 --- a/target/nextflow/workflows/htrnaseq/main.nf +++ b/target/nextflow/workflows/htrnaseq/main.nf @@ -2869,6 +2869,11 @@ meta = [ "path" : "main.nf", "is_executable" : true, "entrypoint" : "run_wf" + }, + { + "type" : "file", + "path" : "/src/config/labels.config", + "dest" : "nextflow_labels.config" } ], "status" : "enabled", @@ -3008,7 +3013,10 @@ meta = [ "cpu200" : "cpus = 200", "cpu500" : "cpus = 500", "cpu1000" : "cpus = 1000" - } + }, + "script" : [ + "includeConfig(\\"nextflow_labels.config\\")" + ] }, "debug" : false, "container" : "docker" @@ -3030,18 +3038,18 @@ meta = [ "engine" : "native|native", "output" : "target/nextflow/workflows/htrnaseq", "viash_version" : "0.9.0-RC6", - "git_commit" : "bdd7091d5b5982b84103a43fece50c608810e0aa", + "git_commit" : "c918fed99aca1cacf3b2c02ca13c2f0af5ecf3bf", "git_remote" : "https://github.com/viash-hub/htrnaseq" }, "package_config" : { "name" : "htrnaseq", "version" : "main_pipeline", - "description" : "Demultiplexing pipeline [WIP]\n", + "description" : "High-throughput pipeline [WIP]\n", "viash_version" : "0.9.0-RC6", "source" : "src", "target" : "target", "config_mods" : [ - ".requirements.commands := ['ps']\n", + ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", ".engines[.type == 'docker'].target_tag := 'main_pipeline'" diff --git a/target/nextflow/workflows/htrnaseq/nextflow.config b/target/nextflow/workflows/htrnaseq/nextflow.config index 6b2f65ef..03476295 100644 --- a/target/nextflow/workflows/htrnaseq/nextflow.config +++ b/target/nextflow/workflows/htrnaseq/nextflow.config @@ -121,4 +121,4 @@ process{ withLabel: cpu1000 { cpus = 1000 } } - +includeConfig("nextflow_labels.config") diff --git a/target/nextflow/workflows/htrnaseq/nextflow_labels.config b/target/nextflow/workflows/htrnaseq/nextflow_labels.config new file mode 100644 index 00000000..9ea88b1b --- /dev/null +++ b/target/nextflow/workflows/htrnaseq/nextflow_labels.config @@ -0,0 +1,43 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + maxMemory = null + + // Resource labels + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } + withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } + +} + +def get_memory(to_compare) { + if (!process.containsKey("maxMemory") || !process.maxMemory) { + return to_compare + } + + try { + if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { + return process.maxMemory + } + else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { + return max_memory as nextflow.util.MemoryUnit + } + else { + return to_compare + } + } catch (all) { + println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" + System.exit(1) + } + } diff --git a/target/nextflow/workflows/utils/groupLanes/.config.vsh.yaml b/target/nextflow/workflows/utils/groupLanes/.config.vsh.yaml index 9975fefc..bd62eb71 100644 --- a/target/nextflow/workflows/utils/groupLanes/.config.vsh.yaml +++ b/target/nextflow/workflows/utils/groupLanes/.config.vsh.yaml @@ -55,6 +55,9 @@ resources: path: "main.nf" is_executable: true entrypoint: "run_wf" +- type: "file" + path: "nextflow_labels.config" + dest: "nextflow_labels.config" description: "N/A\n" info: null status: "enabled" @@ -124,6 +127,8 @@ runners: cpu200: "cpus = 200" cpu500: "cpus = 500" cpu1000: "cpus = 1000" + script: + - "includeConfig(\"nextflow_labels.config\")" debug: false container: "docker" engines: @@ -136,18 +141,20 @@ build_info: output: "target/nextflow/workflows/utils/groupLanes" executable: "target/nextflow/workflows/utils/groupLanes/main.nf" viash_version: "0.9.0-RC6" - git_commit: "bdd7091d5b5982b84103a43fece50c608810e0aa" + git_commit: "c918fed99aca1cacf3b2c02ca13c2f0af5ecf3bf" git_remote: "https://github.com/viash-hub/htrnaseq" package_config: name: "htrnaseq" version: "main_pipeline" - description: "Demultiplexing pipeline [WIP]\n" + description: "High-throughput pipeline [WIP]\n" info: null viash_version: "0.9.0-RC6" source: "src" target: "target" config_mods: - - ".requirements.commands := ['ps']\n" + - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].config.script\ + \ := 'includeConfig(\"nextflow_labels.config\")'\n.resources += {path: '/src/config/labels.config',\ + \ dest: 'nextflow_labels.config'}\n" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - ".engines[.type == 'docker'].target_tag := 'main_pipeline'" diff --git a/target/nextflow/workflows/utils/groupLanes/main.nf b/target/nextflow/workflows/utils/groupLanes/main.nf index c9a31e67..a9a81b27 100644 --- a/target/nextflow/workflows/utils/groupLanes/main.nf +++ b/target/nextflow/workflows/utils/groupLanes/main.nf @@ -2849,6 +2849,11 @@ meta = [ "path" : "main.nf", "is_executable" : true, "entrypoint" : "run_wf" + }, + { + "type" : "file", + "path" : "/src/config/labels.config", + "dest" : "nextflow_labels.config" } ], "description" : "N/A\n", @@ -2925,7 +2930,10 @@ meta = [ "cpu200" : "cpus = 200", "cpu500" : "cpus = 500", "cpu1000" : "cpus = 1000" - } + }, + "script" : [ + "includeConfig(\\"nextflow_labels.config\\")" + ] }, "debug" : false, "container" : "docker" @@ -2943,18 +2951,18 @@ meta = [ "engine" : "native", "output" : "target/nextflow/workflows/utils/groupLanes", "viash_version" : "0.9.0-RC6", - "git_commit" : "bdd7091d5b5982b84103a43fece50c608810e0aa", + "git_commit" : "c918fed99aca1cacf3b2c02ca13c2f0af5ecf3bf", "git_remote" : "https://github.com/viash-hub/htrnaseq" }, "package_config" : { "name" : "htrnaseq", "version" : "main_pipeline", - "description" : "Demultiplexing pipeline [WIP]\n", + "description" : "High-throughput pipeline [WIP]\n", "viash_version" : "0.9.0-RC6", "source" : "src", "target" : "target", "config_mods" : [ - ".requirements.commands := ['ps']\n", + ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", ".engines[.type == 'docker'].target_tag := 'main_pipeline'" diff --git a/target/nextflow/workflows/utils/groupLanes/nextflow.config b/target/nextflow/workflows/utils/groupLanes/nextflow.config index 349068f1..834ceaa6 100644 --- a/target/nextflow/workflows/utils/groupLanes/nextflow.config +++ b/target/nextflow/workflows/utils/groupLanes/nextflow.config @@ -122,4 +122,4 @@ process{ withLabel: cpu1000 { cpus = 1000 } } - +includeConfig("nextflow_labels.config") diff --git a/target/nextflow/workflows/utils/groupLanes/nextflow_labels.config b/target/nextflow/workflows/utils/groupLanes/nextflow_labels.config new file mode 100644 index 00000000..9ea88b1b --- /dev/null +++ b/target/nextflow/workflows/utils/groupLanes/nextflow_labels.config @@ -0,0 +1,43 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + maxMemory = null + + // Resource labels + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } + withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } + +} + +def get_memory(to_compare) { + if (!process.containsKey("maxMemory") || !process.maxMemory) { + return to_compare + } + + try { + if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { + return process.maxMemory + } + else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { + return max_memory as nextflow.util.MemoryUnit + } + else { + return to_compare + } + } catch (all) { + println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" + System.exit(1) + } + } diff --git a/target/nextflow/workflows/utils/groupPairs/.config.vsh.yaml b/target/nextflow/workflows/utils/groupPairs/.config.vsh.yaml index b0f16716..3f866d69 100644 --- a/target/nextflow/workflows/utils/groupPairs/.config.vsh.yaml +++ b/target/nextflow/workflows/utils/groupPairs/.config.vsh.yaml @@ -43,6 +43,9 @@ resources: path: "main.nf" is_executable: true entrypoint: "run_wf" +- type: "file" + path: "nextflow_labels.config" + dest: "nextflow_labels.config" description: "N/A\n" info: null status: "enabled" @@ -112,6 +115,8 @@ runners: cpu200: "cpus = 200" cpu500: "cpus = 500" cpu1000: "cpus = 1000" + script: + - "includeConfig(\"nextflow_labels.config\")" debug: false container: "docker" engines: @@ -124,18 +129,20 @@ build_info: output: "target/nextflow/workflows/utils/groupPairs" executable: "target/nextflow/workflows/utils/groupPairs/main.nf" viash_version: "0.9.0-RC6" - git_commit: "bdd7091d5b5982b84103a43fece50c608810e0aa" + git_commit: "c918fed99aca1cacf3b2c02ca13c2f0af5ecf3bf" git_remote: "https://github.com/viash-hub/htrnaseq" package_config: name: "htrnaseq" version: "main_pipeline" - description: "Demultiplexing pipeline [WIP]\n" + description: "High-throughput pipeline [WIP]\n" info: null viash_version: "0.9.0-RC6" source: "src" target: "target" config_mods: - - ".requirements.commands := ['ps']\n" + - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].config.script\ + \ := 'includeConfig(\"nextflow_labels.config\")'\n.resources += {path: '/src/config/labels.config',\ + \ dest: 'nextflow_labels.config'}\n" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - ".engines[.type == 'docker'].target_tag := 'main_pipeline'" diff --git a/target/nextflow/workflows/utils/groupPairs/main.nf b/target/nextflow/workflows/utils/groupPairs/main.nf index d07a4359..914af9c0 100644 --- a/target/nextflow/workflows/utils/groupPairs/main.nf +++ b/target/nextflow/workflows/utils/groupPairs/main.nf @@ -2835,6 +2835,11 @@ meta = [ "path" : "main.nf", "is_executable" : true, "entrypoint" : "run_wf" + }, + { + "type" : "file", + "path" : "/src/config/labels.config", + "dest" : "nextflow_labels.config" } ], "description" : "N/A\n", @@ -2911,7 +2916,10 @@ meta = [ "cpu200" : "cpus = 200", "cpu500" : "cpus = 500", "cpu1000" : "cpus = 1000" - } + }, + "script" : [ + "includeConfig(\\"nextflow_labels.config\\")" + ] }, "debug" : false, "container" : "docker" @@ -2929,18 +2937,18 @@ meta = [ "engine" : "native", "output" : "target/nextflow/workflows/utils/groupPairs", "viash_version" : "0.9.0-RC6", - "git_commit" : "bdd7091d5b5982b84103a43fece50c608810e0aa", + "git_commit" : "c918fed99aca1cacf3b2c02ca13c2f0af5ecf3bf", "git_remote" : "https://github.com/viash-hub/htrnaseq" }, "package_config" : { "name" : "htrnaseq", "version" : "main_pipeline", - "description" : "Demultiplexing pipeline [WIP]\n", + "description" : "High-throughput pipeline [WIP]\n", "viash_version" : "0.9.0-RC6", "source" : "src", "target" : "target", "config_mods" : [ - ".requirements.commands := ['ps']\n", + ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", ".engines[.type == 'docker'].target_tag := 'main_pipeline'" diff --git a/target/nextflow/workflows/utils/groupPairs/nextflow.config b/target/nextflow/workflows/utils/groupPairs/nextflow.config index 1d9a3358..71d9c1dd 100644 --- a/target/nextflow/workflows/utils/groupPairs/nextflow.config +++ b/target/nextflow/workflows/utils/groupPairs/nextflow.config @@ -122,4 +122,4 @@ process{ withLabel: cpu1000 { cpus = 1000 } } - +includeConfig("nextflow_labels.config") diff --git a/target/nextflow/workflows/utils/groupPairs/nextflow_labels.config b/target/nextflow/workflows/utils/groupPairs/nextflow_labels.config new file mode 100644 index 00000000..9ea88b1b --- /dev/null +++ b/target/nextflow/workflows/utils/groupPairs/nextflow_labels.config @@ -0,0 +1,43 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + maxMemory = null + + // Resource labels + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } + withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } + +} + +def get_memory(to_compare) { + if (!process.containsKey("maxMemory") || !process.maxMemory) { + return to_compare + } + + try { + if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { + return process.maxMemory + } + else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { + return max_memory as nextflow.util.MemoryUnit + } + else { + return to_compare + } + } catch (all) { + println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" + System.exit(1) + } + } diff --git a/target/nextflow/workflows/utils/groupWells/.config.vsh.yaml b/target/nextflow/workflows/utils/groupWells/.config.vsh.yaml index 9d5494f7..66f271ec 100644 --- a/target/nextflow/workflows/utils/groupWells/.config.vsh.yaml +++ b/target/nextflow/workflows/utils/groupWells/.config.vsh.yaml @@ -75,6 +75,9 @@ resources: path: "main.nf" is_executable: true entrypoint: "run_wf" +- type: "file" + path: "nextflow_labels.config" + dest: "nextflow_labels.config" description: "N/A\n" info: null status: "enabled" @@ -144,6 +147,8 @@ runners: cpu200: "cpus = 200" cpu500: "cpus = 500" cpu1000: "cpus = 1000" + script: + - "includeConfig(\"nextflow_labels.config\")" debug: false container: "docker" engines: @@ -156,18 +161,20 @@ build_info: output: "target/nextflow/workflows/utils/groupWells" executable: "target/nextflow/workflows/utils/groupWells/main.nf" viash_version: "0.9.0-RC6" - git_commit: "bdd7091d5b5982b84103a43fece50c608810e0aa" + git_commit: "c918fed99aca1cacf3b2c02ca13c2f0af5ecf3bf" git_remote: "https://github.com/viash-hub/htrnaseq" package_config: name: "htrnaseq" version: "main_pipeline" - description: "Demultiplexing pipeline [WIP]\n" + description: "High-throughput pipeline [WIP]\n" info: null viash_version: "0.9.0-RC6" source: "src" target: "target" config_mods: - - ".requirements.commands := ['ps']\n" + - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].config.script\ + \ := 'includeConfig(\"nextflow_labels.config\")'\n.resources += {path: '/src/config/labels.config',\ + \ dest: 'nextflow_labels.config'}\n" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - ".engines[.type == 'docker'].target_tag := 'main_pipeline'" diff --git a/target/nextflow/workflows/utils/groupWells/main.nf b/target/nextflow/workflows/utils/groupWells/main.nf index 47068ff2..93edf4e9 100644 --- a/target/nextflow/workflows/utils/groupWells/main.nf +++ b/target/nextflow/workflows/utils/groupWells/main.nf @@ -2873,6 +2873,11 @@ meta = [ "path" : "main.nf", "is_executable" : true, "entrypoint" : "run_wf" + }, + { + "type" : "file", + "path" : "/src/config/labels.config", + "dest" : "nextflow_labels.config" } ], "description" : "N/A\n", @@ -2949,7 +2954,10 @@ meta = [ "cpu200" : "cpus = 200", "cpu500" : "cpus = 500", "cpu1000" : "cpus = 1000" - } + }, + "script" : [ + "includeConfig(\\"nextflow_labels.config\\")" + ] }, "debug" : false, "container" : "docker" @@ -2967,18 +2975,18 @@ meta = [ "engine" : "native", "output" : "target/nextflow/workflows/utils/groupWells", "viash_version" : "0.9.0-RC6", - "git_commit" : "bdd7091d5b5982b84103a43fece50c608810e0aa", + "git_commit" : "c918fed99aca1cacf3b2c02ca13c2f0af5ecf3bf", "git_remote" : "https://github.com/viash-hub/htrnaseq" }, "package_config" : { "name" : "htrnaseq", "version" : "main_pipeline", - "description" : "Demultiplexing pipeline [WIP]\n", + "description" : "High-throughput pipeline [WIP]\n", "viash_version" : "0.9.0-RC6", "source" : "src", "target" : "target", "config_mods" : [ - ".requirements.commands := ['ps']\n", + ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", ".engines[.type == 'docker'].target_tag := 'main_pipeline'" diff --git a/target/nextflow/workflows/utils/groupWells/nextflow.config b/target/nextflow/workflows/utils/groupWells/nextflow.config index c9e46d7d..245d4336 100644 --- a/target/nextflow/workflows/utils/groupWells/nextflow.config +++ b/target/nextflow/workflows/utils/groupWells/nextflow.config @@ -122,4 +122,4 @@ process{ withLabel: cpu1000 { cpus = 1000 } } - +includeConfig("nextflow_labels.config") diff --git a/target/nextflow/workflows/utils/groupWells/nextflow_labels.config b/target/nextflow/workflows/utils/groupWells/nextflow_labels.config new file mode 100644 index 00000000..9ea88b1b --- /dev/null +++ b/target/nextflow/workflows/utils/groupWells/nextflow_labels.config @@ -0,0 +1,43 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + maxMemory = null + + // Resource labels + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } + withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } + +} + +def get_memory(to_compare) { + if (!process.containsKey("maxMemory") || !process.maxMemory) { + return to_compare + } + + try { + if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { + return process.maxMemory + } + else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { + return max_memory as nextflow.util.MemoryUnit + } + else { + return to_compare + } + } catch (all) { + println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" + System.exit(1) + } + } diff --git a/target/nextflow/workflows/utils/splitWells/.config.vsh.yaml b/target/nextflow/workflows/utils/splitWells/.config.vsh.yaml index fece08a5..942d7a1c 100644 --- a/target/nextflow/workflows/utils/splitWells/.config.vsh.yaml +++ b/target/nextflow/workflows/utils/splitWells/.config.vsh.yaml @@ -61,6 +61,9 @@ resources: path: "main.nf" is_executable: true entrypoint: "run_wf" +- type: "file" + path: "nextflow_labels.config" + dest: "nextflow_labels.config" description: "N/A\n" info: null status: "enabled" @@ -130,6 +133,8 @@ runners: cpu200: "cpus = 200" cpu500: "cpus = 500" cpu1000: "cpus = 1000" + script: + - "includeConfig(\"nextflow_labels.config\")" debug: false container: "docker" engines: @@ -142,18 +147,20 @@ build_info: output: "target/nextflow/workflows/utils/splitWells" executable: "target/nextflow/workflows/utils/splitWells/main.nf" viash_version: "0.9.0-RC6" - git_commit: "bdd7091d5b5982b84103a43fece50c608810e0aa" + git_commit: "c918fed99aca1cacf3b2c02ca13c2f0af5ecf3bf" git_remote: "https://github.com/viash-hub/htrnaseq" package_config: name: "htrnaseq" version: "main_pipeline" - description: "Demultiplexing pipeline [WIP]\n" + description: "High-throughput pipeline [WIP]\n" info: null viash_version: "0.9.0-RC6" source: "src" target: "target" config_mods: - - ".requirements.commands := ['ps']\n" + - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].config.script\ + \ := 'includeConfig(\"nextflow_labels.config\")'\n.resources += {path: '/src/config/labels.config',\ + \ dest: 'nextflow_labels.config'}\n" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - ".engines[.type == 'docker'].target_tag := 'main_pipeline'" diff --git a/target/nextflow/workflows/utils/splitWells/main.nf b/target/nextflow/workflows/utils/splitWells/main.nf index b7f27c05..ebc5ffc6 100644 --- a/target/nextflow/workflows/utils/splitWells/main.nf +++ b/target/nextflow/workflows/utils/splitWells/main.nf @@ -2856,6 +2856,11 @@ meta = [ "path" : "main.nf", "is_executable" : true, "entrypoint" : "run_wf" + }, + { + "type" : "file", + "path" : "/src/config/labels.config", + "dest" : "nextflow_labels.config" } ], "description" : "N/A\n", @@ -2932,7 +2937,10 @@ meta = [ "cpu200" : "cpus = 200", "cpu500" : "cpus = 500", "cpu1000" : "cpus = 1000" - } + }, + "script" : [ + "includeConfig(\\"nextflow_labels.config\\")" + ] }, "debug" : false, "container" : "docker" @@ -2950,18 +2958,18 @@ meta = [ "engine" : "native", "output" : "target/nextflow/workflows/utils/splitWells", "viash_version" : "0.9.0-RC6", - "git_commit" : "bdd7091d5b5982b84103a43fece50c608810e0aa", + "git_commit" : "c918fed99aca1cacf3b2c02ca13c2f0af5ecf3bf", "git_remote" : "https://github.com/viash-hub/htrnaseq" }, "package_config" : { "name" : "htrnaseq", "version" : "main_pipeline", - "description" : "Demultiplexing pipeline [WIP]\n", + "description" : "High-throughput pipeline [WIP]\n", "viash_version" : "0.9.0-RC6", "source" : "src", "target" : "target", "config_mods" : [ - ".requirements.commands := ['ps']\n", + ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", ".engines[.type == 'docker'].target_tag := 'main_pipeline'" diff --git a/target/nextflow/workflows/utils/splitWells/nextflow.config b/target/nextflow/workflows/utils/splitWells/nextflow.config index 6d9b8a73..810864cf 100644 --- a/target/nextflow/workflows/utils/splitWells/nextflow.config +++ b/target/nextflow/workflows/utils/splitWells/nextflow.config @@ -122,4 +122,4 @@ process{ withLabel: cpu1000 { cpus = 1000 } } - +includeConfig("nextflow_labels.config") diff --git a/target/nextflow/workflows/utils/splitWells/nextflow_labels.config b/target/nextflow/workflows/utils/splitWells/nextflow_labels.config new file mode 100644 index 00000000..9ea88b1b --- /dev/null +++ b/target/nextflow/workflows/utils/splitWells/nextflow_labels.config @@ -0,0 +1,43 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + maxMemory = null + + // Resource labels + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } + withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } + +} + +def get_memory(to_compare) { + if (!process.containsKey("maxMemory") || !process.maxMemory) { + return to_compare + } + + try { + if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { + return process.maxMemory + } + else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { + return max_memory as nextflow.util.MemoryUnit + } + else { + return to_compare + } + } catch (all) { + println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" + System.exit(1) + } + } diff --git a/target/nextflow/workflows/well_demultiplex/.config.vsh.yaml b/target/nextflow/workflows/well_demultiplex/.config.vsh.yaml index 1744485f..13fe1d94 100644 --- a/target/nextflow/workflows/well_demultiplex/.config.vsh.yaml +++ b/target/nextflow/workflows/well_demultiplex/.config.vsh.yaml @@ -52,6 +52,9 @@ resources: path: "main.nf" is_executable: true entrypoint: "run_wf" +- type: "file" + path: "nextflow_labels.config" + dest: "nextflow_labels.config" description: "Demultiplexing on well level" test_resources: - type: "nextflow_script" @@ -137,6 +140,8 @@ runners: cpu200: "cpus = 200" cpu500: "cpus = 500" cpu1000: "cpus = 1000" + script: + - "includeConfig(\"nextflow_labels.config\")" debug: false container: "docker" engines: @@ -151,20 +156,22 @@ build_info: output: "target/nextflow/workflows/well_demultiplex" executable: "target/nextflow/workflows/well_demultiplex/main.nf" viash_version: "0.9.0-RC6" - git_commit: "bdd7091d5b5982b84103a43fece50c608810e0aa" + git_commit: "c918fed99aca1cacf3b2c02ca13c2f0af5ecf3bf" git_remote: "https://github.com/viash-hub/htrnaseq" dependencies: - "target/dependencies/vsh/vsh/biobox/v0.1.0/nextflow/cutadapt" package_config: name: "htrnaseq" version: "main_pipeline" - description: "Demultiplexing pipeline [WIP]\n" + description: "High-throughput pipeline [WIP]\n" info: null viash_version: "0.9.0-RC6" source: "src" target: "target" config_mods: - - ".requirements.commands := ['ps']\n" + - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].config.script\ + \ := 'includeConfig(\"nextflow_labels.config\")'\n.resources += {path: '/src/config/labels.config',\ + \ dest: 'nextflow_labels.config'}\n" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - ".engines[.type == 'docker'].target_tag := 'main_pipeline'" diff --git a/target/nextflow/workflows/well_demultiplex/main.nf b/target/nextflow/workflows/well_demultiplex/main.nf index 74d6207a..6632edc8 100644 --- a/target/nextflow/workflows/well_demultiplex/main.nf +++ b/target/nextflow/workflows/well_demultiplex/main.nf @@ -2845,6 +2845,11 @@ meta = [ "path" : "main.nf", "is_executable" : true, "entrypoint" : "run_wf" + }, + { + "type" : "file", + "path" : "/src/config/labels.config", + "dest" : "nextflow_labels.config" } ], "description" : "Demultiplexing on well level", @@ -2947,7 +2952,10 @@ meta = [ "cpu200" : "cpus = 200", "cpu500" : "cpus = 500", "cpu1000" : "cpus = 1000" - } + }, + "script" : [ + "includeConfig(\\"nextflow_labels.config\\")" + ] }, "debug" : false, "container" : "docker" @@ -2969,18 +2977,18 @@ meta = [ "engine" : "native|native", "output" : "target/nextflow/workflows/well_demultiplex", "viash_version" : "0.9.0-RC6", - "git_commit" : "bdd7091d5b5982b84103a43fece50c608810e0aa", + "git_commit" : "c918fed99aca1cacf3b2c02ca13c2f0af5ecf3bf", "git_remote" : "https://github.com/viash-hub/htrnaseq" }, "package_config" : { "name" : "htrnaseq", "version" : "main_pipeline", - "description" : "Demultiplexing pipeline [WIP]\n", + "description" : "High-throughput pipeline [WIP]\n", "viash_version" : "0.9.0-RC6", "source" : "src", "target" : "target", "config_mods" : [ - ".requirements.commands := ['ps']\n", + ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", ".engines[.type == 'docker'].target_tag := 'main_pipeline'" diff --git a/target/nextflow/workflows/well_demultiplex/nextflow.config b/target/nextflow/workflows/well_demultiplex/nextflow.config index f9471444..70bd4994 100644 --- a/target/nextflow/workflows/well_demultiplex/nextflow.config +++ b/target/nextflow/workflows/well_demultiplex/nextflow.config @@ -122,4 +122,4 @@ process{ withLabel: cpu1000 { cpus = 1000 } } - +includeConfig("nextflow_labels.config") diff --git a/target/nextflow/workflows/well_demultiplex/nextflow_labels.config b/target/nextflow/workflows/well_demultiplex/nextflow_labels.config new file mode 100644 index 00000000..9ea88b1b --- /dev/null +++ b/target/nextflow/workflows/well_demultiplex/nextflow_labels.config @@ -0,0 +1,43 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + maxMemory = null + + // Resource labels + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } + withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } + +} + +def get_memory(to_compare) { + if (!process.containsKey("maxMemory") || !process.maxMemory) { + return to_compare + } + + try { + if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { + return process.maxMemory + } + else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { + return max_memory as nextflow.util.MemoryUnit + } + else { + return to_compare + } + } catch (all) { + println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" + System.exit(1) + } + }