vsh/htrnaseq
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Build branch main with version main (65dd41d)

Browse Source 
Build pipeline: viash-hub.htrnaseq.main-vhms8

Source commit: 65dd41d8b1

Source message: Optimize spawning of processes
This commit is contained in:
CI
2024-11-05 17:26:35 +00:00
parent b8abf8c490
commit f2ff92c6ac
127 changed files with 29560 additions and 1004 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
target/nextflow/parallel_map/.config.vsh.yaml
View File

@@ -110,6 +110,8 @@ resources:
- type: "bash_script"
path: "script.sh"
is_executable: true
- type: "file"
path: "STAR"
- type: "file"
path: "nextflow_labels.config"
dest: "nextflow_labels.config"
@@ -205,26 +207,25 @@ engines:
- type: "apt"
packages:
- "procps"
- "gzip"
- "bzip2"
- "parallel"
- "wget"
- "automake"
- "make"
- "gcc"
- "g++"
- "zlib1g-dev"
- "unzip"
- "xxd"
- "parallel"
- "file"
interactive: false
- type: "docker"
run:
- "wget -O $STAR_TARGET $STAR_SOURCE && \\\n unzip $STAR_TARGET -d /tmp && \\\
\n mv /tmp/STAR_$STAR_VERSION/Linux_x86_64_static/STAR /usr/local/bin/$STAR_BINARY\
\ && \\\n chmod +x /usr/local/bin/$STAR_BINARY && \\\n rm $STAR_TARGET &&\
\ rm -rf /tmp/STAR_$STAR_VERSION\n"
copy:
- "STAR /usr/local/bin/$STAR_BINARY"
build_args:
- "STAR_V=2.7.6a"
env:
- "STAR_VERSION \"2.7.11b\""
- "STAR_SOURCE \"https://github.com/alexdobin/STAR/releases/download/$STAR_VERSION/STAR_$STAR_VERSION.zip\""
- "STAR_TARGET \"/tmp/star.zip\""
- "STAR_BINARY \"STAR\""
- "STAR_SOURCE=\"https://github.com/alexdobin/STAR/archive/refs/tags/$STAR_V.tar.gz\""
- "STAR_TARGET=\"/app/star-$STAR_V.tar.gz\""
- "STAR_INSTALL_DIR=\"/app/STAR-$STAR_V\""
- "STAR_BINARY=STAR"
entrypoint: []
cmd: null
- type: "native"
@@ -235,15 +236,18 @@ build_info:
engine: "docker|native"
output: "target/nextflow/parallel_map"
executable: "target/nextflow/parallel_map/main.nf"
viash_version: "0.9.0-RC7"
git_commit: "cf9797232db1306bfd5696287928cababe317d99"
git_remote: "https://x-access-token:ghs_KjB7pWu8DQM3iFulLu7RI06qnt5K8S1A0eaE@github.com/viash-hub/htrnaseq"
viash_version: "0.9.0"
git_commit: "65dd41d8b1b4a307735c72320c96c0880c75f17f"
git_remote: "https://x-access-token:ghs_McZDF0yobnnHmOEb2Q4JaaB3pzr9mz1VbIOs@github.com/viash-hub/htrnaseq"
package_config:
name: "htrnaseq"
version: "main"
description: "High-throughput pipeline [WIP]\n"
info: null
viash_version: "0.9.0-RC7"
info:
test_resources:
- path: "gs://viash-hub-test-data/htrnaseq/v1/"
dest: "resources_test"
viash_version: "0.9.0"
source: "src"
target: "target"
config_mods:

BIN
target/nextflow/parallel_map/STAR Executable file
View File

Binary file not shown.

74
target/nextflow/parallel_map/main.nf
View File

@@ -1,8 +1,8 @@
// parallel_map main
//
// This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a
// derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from
// Data Intuitive.
// This wrapper script is auto-generated by viash 0.9.0 and is thus a derivative
// work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data
// Intuitive.
//
// The component may contain files which fall under a different license. The
// authors of this component should specify the license in the header of such
@@ -1728,7 +1728,9 @@ def publishStates(Map args) {
def yamlFilename = yamlTemplate_
.replaceAll('\\$id', id_)
.replaceAll('\\$\\{id\\}', id_)
.replaceAll('\\$key', key_)
.replaceAll('\\$\\{key\\}', key_)
// TODO: do the pathnames in state_ match up with the outputFilenames_?
@@ -1799,7 +1801,9 @@ def publishStatesByConfig(Map args) {
def yamlTemplate = params.containsKey("output_state") ? params.output_state : '$id.$key.state.yaml'
def yamlFilename = yamlTemplate
.replaceAll('\\$id', id_)
.replaceAll('\\$\\{id\\}', id_)
.replaceAll('\\$key', key_)
.replaceAll('\\$\\{key\\}', key_)
def yamlDir = java.nio.file.Paths.get(yamlFilename).getParent()
// the processed state is a list of [key, value, inputPath, outputFilename] tuples, where
@@ -1841,7 +1845,9 @@ def publishStatesByConfig(Map args) {
// instantiate the template
def filename = filenameTemplate
.replaceAll('\\$id', id_)
.replaceAll('\\$\\{id\\}', id_)
.replaceAll('\\$key', key_)
.replaceAll('\\$\\{key\\}', key_)
if (par.multiple) {
// if the parameter is multiple: true, the filename
// should contain a wildcard '*' that is replaced with
@@ -2937,6 +2943,10 @@ meta = [
"path" : "script.sh",
"is_executable" : true
},
{
"type" : "file",
"path" : "STAR"
},
{
"type" : "file",
"path" : "/src/config/labels.config",
@@ -3051,27 +3061,30 @@ meta = [
"type" : "apt",
"packages" : [
"procps",
"gzip",
"bzip2",
"parallel",
"wget",
"automake",
"make",
"gcc",
"g++",
"zlib1g-dev",
"unzip",
"xxd",
"parallel",
"file"
],
"interactive" : false
},
{
"type" : "docker",
"run" : [
"wget -O $STAR_TARGET $STAR_SOURCE && \\\\\n unzip $STAR_TARGET -d /tmp && \\\\\n mv /tmp/STAR_$STAR_VERSION/Linux_x86_64_static/STAR /usr/local/bin/$STAR_BINARY && \\\\\n chmod +x /usr/local/bin/$STAR_BINARY && \\\\\n rm $STAR_TARGET && rm -rf /tmp/STAR_$STAR_VERSION\n"
"copy" : [
"STAR /usr/local/bin/$STAR_BINARY"
],
"build_args" : [
"STAR_V=2.7.6a"
],
"env" : [
"STAR_VERSION \\"2.7.11b\\"",
"STAR_SOURCE \\"https://github.com/alexdobin/STAR/releases/download/$STAR_VERSION/STAR_$STAR_VERSION.zip\\"",
"STAR_TARGET \\"/tmp/star.zip\\"",
"STAR_BINARY \\"STAR\\""
"STAR_SOURCE=\\"https://github.com/alexdobin/STAR/archive/refs/tags/$STAR_V.tar.gz\\"",
"STAR_TARGET=\\"/app/star-$STAR_V.tar.gz\\"",
"STAR_INSTALL_DIR=\\"/app/STAR-$STAR_V\\"",
"STAR_BINARY=STAR"
]
}
]
@@ -3086,15 +3099,23 @@ meta = [
"runner" : "nextflow",
"engine" : "docker|native",
"output" : "target/nextflow/parallel_map",
"viash_version" : "0.9.0-RC7",
"git_commit" : "cf9797232db1306bfd5696287928cababe317d99",
"git_remote" : "https://x-access-token:ghs_KjB7pWu8DQM3iFulLu7RI06qnt5K8S1A0eaE@github.com/viash-hub/htrnaseq"
"viash_version" : "0.9.0",
"git_commit" : "65dd41d8b1b4a307735c72320c96c0880c75f17f",
"git_remote" : "https://x-access-token:ghs_McZDF0yobnnHmOEb2Q4JaaB3pzr9mz1VbIOs@github.com/viash-hub/htrnaseq"
},
"package_config" : {
"name" : "htrnaseq",
"version" : "main",
"description" : "High-throughput pipeline [WIP]\n",
"viash_version" : "0.9.0-RC7",
"info" : {
"test_resources" : [
{
"path" : "gs://viash-hub-test-data/htrnaseq/v1/",
"dest" : "resources_test"
}
]
},
"viash_version" : "0.9.0",
"source" : "src",
"target" : "target",
"config_mods" : [
@@ -3516,7 +3537,11 @@ def vdsl3WorkflowFactory(Map args, Map meta, String rawScript) {
val = val.join(par.multiple_sep)
}
if (par.direction == "output" && par.type == "file") {
val = val.replaceAll('\\$id', id).replaceAll('\\$key', key)
val = val
.replaceAll('\\$id', id)
.replaceAll('\\$\\{id\\}', id)
.replaceAll('\\$key', key)
.replaceAll('\\$\\{key\\}', key)
}
[parName, val]
}
@@ -3647,7 +3672,8 @@ def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) {
def createParentStr = meta.config.allArguments
.findAll { it.type == "file" && it.direction == "output" && it.create_parent }
.collect { par ->
"\${ args.containsKey(\"${par.plainName}\") ? \"mkdir_parent \\\"\" + (args[\"${par.plainName}\"] instanceof String ? args[\"${par.plainName}\"] : args[\"${par.plainName}\"].join('\" \"')) + \"\\\"\" : \"\" }"
def contents = "args[\"${par.plainName}\"] instanceof List ? args[\"${par.plainName}\"].join('\" \"') : args[\"${par.plainName}\"]"
"\${ args.containsKey(\"${par.plainName}\") ? \"mkdir_parent '\" + escapeText(${contents}) + \"'\" : \"\" }"
}
.join("\n")
@@ -3655,8 +3681,8 @@ def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) {
def inputFileExports = meta.config.allArguments
.findAll { it.type == "file" && it.direction.toLowerCase() == "input" }
.collect { par ->
def viash_par_contents = "(viash_par_${par.plainName} instanceof List ? viash_par_${par.plainName}.join(\"${par.multiple_sep}\") : viash_par_${par.plainName})"
"\n\${viash_par_${par.plainName}.empty ? \"\" : \"export VIASH_PAR_${par.plainName.toUpperCase()}=\\\"\" + ${viash_par_contents} + \"\\\"\"}"
def contents = "viash_par_${par.plainName} instanceof List ? viash_par_${par.plainName}.join(\"${par.multiple_sep}\") : viash_par_${par.plainName}"
"\n\${viash_par_${par.plainName}.empty ? \"\" : \"export VIASH_PAR_${par.plainName.toUpperCase()}='\" + escapeText(${contents}) + \"'\"}"
}
// NOTE: if using docker, use /tmp instead of tmpDir!
@@ -3693,6 +3719,7 @@ def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) {
def procStr =
"""nextflow.enable.dsl=2
|
|def escapeText = { s -> s.toString().replaceAll("'", "'\\\"'\\\"'") }
|process $procKey {$drctvStrs
|input:
| tuple val(id)$inputPaths, val(args), path(resourcesDir, stageAs: ".viash_meta_resources")
@@ -3704,10 +3731,9 @@ def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) {
|$stub
|\"\"\"
|script:$assertStr
|def escapeText = { s -> s.toString().replaceAll('([`"])', '\\\\\\\\\$1') }
|def parInject = args
| .findAll{key, value -> value != null}
| .collect{key, value -> "export VIASH_PAR_\${key.toUpperCase()}=\\\"\${escapeText(value)}\\\""}
| .collect{key, value -> "export VIASH_PAR_\${key.toUpperCase()}='\${escapeText(value)}'"}
| .join("\\n")
|\"\"\"
|# meta exports

86
target/nextflow/parallel_map/nextflow_labels.config
View File

@@ -1,26 +1,88 @@
executor {
$k8s {
submitRateLimit = '10sec'
pollInterval = '1 sec'
}
}
process {
// Default resources for components that hardly do any processing
memory = { 2.GB * task.attempt }
cpus = 1
container = 'nextflow/bash:latest'
// default resources
memory = { 8.Gb * task.attempt }
cpus = 8
maxForks = 36
// Retry for exit codes that have something to do with memory issues
errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' }
maxRetries = 3
maxMemory = null
maxMemory = 192.GB
// Resource labels
withLabel: singlecpu { cpus = 1 }
withLabel: lowcpu { cpus = 4 }
withLabel: midcpu { cpus = 10 }
withLabel: highcpu { cpus = 20 }
withLabel: verylowcpu { cpus = 2 }
withLabel: lowcpu { cpus = 8 }
withLabel: midcpu { cpus = 16 }
withLabel: highcpu { cpus = 32 }
withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } }
withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } }
withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } }
withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } }
withLabel: verylowmem { memory = { get_memory( 4.GB * task.attempt ) } }
withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } }
withLabel: midmem { memory = { get_memory( 16.GB * task.attempt ) } }
withLabel: highmem { memory = { get_memory( 64.GB * task.attempt ) } }
}
profiles {
// detect tempdir
tempDir = java.nio.file.Paths.get(
System.getenv('NXF_TEMP') ?:
System.getenv('VIASH_TEMP') ?:
System.getenv('TEMPDIR') ?:
System.getenv('TMPDIR') ?:
'/tmp'
).toAbsolutePath()
mount_temp {
docker.temp = tempDir
podman.temp = tempDir
charliecloud.temp = tempDir
}
no_publish {
process {
withName: '.*' {
publishDir = [
enabled: false
]
}
}
}
docker {
docker.fixOwnership = true
docker.enabled = true
// docker.userEmulation = true
singularity.enabled = false
podman.enabled = false
shifter.enabled = false
charliecloud.enabled = false
}
local {
// This config is for local processing.
process {
maxMemory = 25.GB
withLabel: verylowcpu { cpus = 2 }
withLabel: lowcpu { cpus = 4 }
withLabel: midcpu { cpus = 6 }
withLabel: highcpu { cpus = 12 }
withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } }
withLabel: midmem { memory = { get_memory( 12.GB * task.attempt ) } }
withLabel: highmem { memory = { get_memory( 20.GB * task.attempt ) } }
}
}
}
def get_memory(to_compare) {
if (!process.containsKey("maxMemory") || !process.maxMemory) {
return to_compare